Update tests to properly skip, add experimental windows CI, remove co…

…v from macos test (fails)
Farama-Foundation · Nov 14, 2023 · bec6a85 · bec6a85
1 parent 99e3c9c
commit bec6a85
Show file tree

Hide file tree

Showing 5 changed files with 240 additions and 108 deletions.
diff --git a/.github/workflows/macos-test.yml b/.github/workflows/macos-test.yml
@@ -33,4 +33,4 @@ jobs:
           pip install dist/*.tar.gz
       - name: Release Test
         run: |
-          pytest -v -n auto tests/ --cov=chatarena--cov-report term
+          pytest -v -n auto tests/
diff --git a/.github/workflows/windows-test.yml b/.github/workflows/windows-test.yml
@@ -0,0 +1,36 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+---
+name: Windows tests
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  linux-test:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -e '.[all]'
+      - name: Source distribution test
+        run: |
+          python -m pip install --upgrade build
+          python -m build --sdist
+          pip install dist/*.tar.gz
+      - name: Release Test
+        run: |
+          pytest -v -n auto tests/
diff --git a/chatarena/__init__.py b/chatarena/__init__.py
@@ -0,0 +1,6 @@
+import os
+
+ROOT_DIR = (
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.path.sep
+)
+EXAMPLES_DIR = os.path.join(ROOT_DIR, "examples")
diff --git a/tests/unit/test_arena.py b/tests/unit/test_arena.py
@@ -2,16 +2,17 @@
 import unittest
 from unittest import TestCase
 
+from chatarena import EXAMPLES_DIR
 from chatarena.arena import Arena
 
 
 class TestArena(TestCase):
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
     def test_arena_1(self):
-        unittest.skipIf(
-            not os.getenv("OPENAI_API_KEY"),
-            "OpenAI API key must be set to run this test.",
-        )
-        arena = Arena.from_config("examples/nlp-classroom.json")
+        arena = Arena.from_config(os.path.join(EXAMPLES_DIR, "nlp-classroom.json"))
 
         print("=== Step 1 ===")
         arena.step()
@@ -27,80 +28,147 @@ def test_arena_1(self):
 
         self.assertTrue(True)
 
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
     def test_arena_2(self):
-        unittest.skipIf(
-            not os.getenv("OPENAI_API_KEY"),
-            "OpenAI API key must be set to run this test.",
-        )
-        arena = Arena.from_config("examples/nlp-classroom.json")
+        arena = Arena.from_config(os.path.join(EXAMPLES_DIR, "nlp-classroom.json"))
 
         arena.run(num_steps=10)
         arena.environment.print()
 
         self.assertTrue(True)
 
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
     def test_arena_3(self):
-        unittest.skipIf(
-            not os.getenv("OPENAI_API_KEY"),
-            "OpenAI API key must be set to run this test.",
+        arena = Arena.from_config(os.path.join(EXAMPLES_DIR, "tic-tac-toe.json"))
+
+        for i in range(1, 10):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
+
+        self.assertTrue(True)
+
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_4(self):
+        arena = Arena.from_config(os.path.join(EXAMPLES_DIR, "chameleon.json"))
+        for i in range(1, 10):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
+
+        self.assertTrue(True)
+
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_5(self):
+        arena = Arena.from_config(
+            os.path.join(EXAMPLES_DIR, "rock-paper-scissors.json")
+        )
+        for i in range(1, 10):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
+
+        self.assertTrue(True)
+
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_6(self):
+        arena = Arena.from_config(
+            os.path.join(EXAMPLES_DIR, "nlp-classroom-3players.json")
+        )
+        for i in range(1, 10):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
+
+        self.assertTrue(True)
+
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_7(self):
+        arena = Arena.from_config(os.path.join(EXAMPLES_DIR, "pettingzoo_chess.json"))
+        for i in range(1, 2):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
+
+        self.assertTrue(True)
+
+    @unittest.skipIf(
+        not os.environ.get("ANTHROPIC_API_KEY"),
+        "Anthropic API key must be set to run this test.",
+    )
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_8(self):
+        arena = Arena.from_config(
+            os.path.join(EXAMPLES_DIR, "chatgpt_claude_ai_collaboration.json")
         )
-        arena = Arena.from_config("examples/tic-tac-toe.json")
+        for i in range(1, 10):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
 
+        self.assertTrue(True)
+
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_9(self):
+        arena = Arena.from_config(os.path.join(EXAMPLES_DIR, "interview.json"))
+        for i in range(1, 10):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
+
+        self.assertTrue(True)
+
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_10(self):
+        arena = Arena.from_config(os.path.join(EXAMPLES_DIR, "prisoners_dilemma.json"))
         for i in range(1, 10):
             print(f"=== Step {i} ===")
             arena.step()
             arena.environment.print()
 
         self.assertTrue(True)
 
-    # def test_arena_4(self):
-    #     with open("examples/nlp-classroom.json", "r") as fp:
-    #         config = json.load(fp)
-    #     arena = Arena.from_config(config)
-    #     arena.launch_gradio()
-    #
-    #     self.assertTrue(True)
-    #
-    # def test_arena_5(self):
-    #     with open("examples/tic-tac-toe.json", "r") as fp:
-    #         config = json.load(fp)
-    #     arena = Arena.from_config(config)
-    #     arena.launch_gradio()
-    #
-    #     self.assertTrue(True)
-    #
-    # def test_arena_6(self):
-    #     with open("examples/nlp-classroom-gpt4.json", "r") as fp:
-    #         config = json.load(fp)
-    #     arena = Arena.from_config(config)
-    #     arena.launch_gradio()
-    #
-    #     self.assertTrue(True)
-    #
-    # def test_arena_7(self):
-    #     with open("examples/tic-tac-toe-gpt4.json", "r") as fp:
-    #         config = json.load(fp)
-    #     arena = Arena.from_config(config)
-    #     arena.launch_gradio()
-    #
-    #     self.assertTrue(True)
-    #
-    # def test_arena_8(self):
-    #     with open("examples/nlp-classroom-3players.json", "r") as fp:
-    #         config = json.load(fp)
-    #     arena = Arena.from_config(config)
-    #     arena.launch_gradio()
-    #
-    #     self.assertTrue(True)
-    #
-    #
-    # def test_arena_9(self):
-    #     with open("examples/rock-paper-scissors.json", "r") as fp:
-    #         config = json.load(fp)
-    #     arena = Arena.from_config(config)
-    #     arena.launch_gradio()
-    #
-    #     self.assertTrue(True)
+    @unittest.skipIf(
+        not os.getenv("OPENAI_API_KEY"),
+        "OpenAI API key must be set to run this test.",
+    )
+    def test_arena_11(self):
+        arena = Arena.from_config(
+            os.path.join(EXAMPLES_DIR, "pettingzoo_tictactoe.json")
+        )
+        for i in range(1, 2):
+            print(f"=== Step {i} ===")
+            arena.step()
+            arena.environment.print()
+
+        self.assertTrue(True)
 
 
 if __name__ == "__main__":