Merge branch 'master' into video-inference

Author: deigen

.github/workflows/run_tests.yml

@@ -24,7 +24,6 @@ on:
 
 jobs:
   build:
-
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
@@ -51,6 +50,18 @@ jobs:
         pip install genbadge[coverage]
     - name: Run static analysis lint
       uses: pre-commit/action@v3.0.0
+    - name: Check if Dockerfile.template changed
+      id: check_dockerfile_change
+      shell: bash
+      run: |
+        # We compare the files changed between the previous commit SHA (github.event.before)
+        # and the current commit SHA (github.event.after).
+        # If Dockerfile.template is in that list, we set changed=true
+        if git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | grep -q 'clarifai/runners/dockerfile_template/Dockerfile.template'; then
+          echo "changed=true" >> $GITHUB_OUTPUT
+        else
+          echo "changed=false" >> $GITHUB_OUTPUT
+        fi
     - name: Prepare the API keys & Run pytest
       env:
         CLARIFAI_USER_EMAIL_SECURE_HOSTING: ${{ secrets.CLARIFAI_USER_EMAIL_SECURE_HOSTING }}
@@ -71,11 +82,25 @@ jobs:
         export CLARIFAI_USER_ID="$(python scripts/key_for_tests.py --get-userid)"
         export CLARIFAI_PAT="$(python scripts/key_for_tests.py --create-pat)"
         if [ "${{ github.event.pull_request.head.repo.full_name }}" = "${{ github.repository }}" ]; then
-          pytest --cov=. --cov-report=xml:coverage/coverage.cobertura.xml
+          # Run coverage for all but the container test
+          pytest --cov=. --cov-report=xml:coverage/coverage.cobertura.xml --ignore=tests/runners/test_model_run_locally-container.py
+
+          # Conditionally run the container test
+          if [ "${{ steps.check_dockerfile_change.outputs.changed }}" = "true" ]; then
+            pytest --cov=. --cov-append --cov-report=xml:coverage/coverage.cobertura.xml tests/runners/test_model_run_locally-container.py
+          fi
         elif [ "${{ inputs.PERIODIC_CHECKS }}" = "true" ]; then
-          pytest -m "not coverage_only" --cov=. --cov-report=xml:coverage/coverage.cobertura.xml
+          pytest -m "not coverage_only" --cov=. --cov-report=xml:coverage/coverage.cobertura.xml --ignore=tests/runners/test_model_run_locally-container.py
+
+          if [ "${{ steps.check_dockerfile_change.outputs.changed }}" = "true" ]; then
+            pytest -m "not coverage_only" --cov=. --cov-append --cov-report=xml:coverage/coverage.cobertura.xml tests/runners/test_model_run_locally-container.py
+          fi
         else
-          pytest -m "not requires_secrets" --cov=. --cov-report=xml:coverage/coverage.cobertura.xml
+          pytest -m "not requires_secrets" --cov=. --cov-report=xml:coverage/coverage.cobertura.xml --ignore=tests/runners/test_model_run_locally-container.py
+
+          if [ "${{ steps.check_dockerfile_change.outputs.changed }}" = "true" ]; then
+            pytest -m "not requires_secrets" --cov=. --cov-append --cov-report=xml:coverage/coverage.cobertura.xml tests/runners/test_model_run_locally-container.py
+          fi
         fi
     - name: Code Coverage Report
       if: runner.os == 'Linux' && matrix.python-version == '3.11'

clarifai/runners/models/model_builder.py

@@ -477,7 +477,8 @@ def get_model_version_proto(self):
         for concept in labels:
           concept_proto = json_format.ParseDict(concept, resources_pb2.Concept())
           model_version_proto.output_info.data.concepts.append(concept_proto)
-      else:
+      elif self.config.get("checkpoints") and HuggingFaceLoader.validate_concept(
+          self.checkpoint_path):
         labels = HuggingFaceLoader.fetch_labels(self.checkpoint_path)
         logger.info(f"Found {len(labels)} concepts from the model checkpoints.")
         # sort the concepts by id and then update the config file

clarifai/runners/utils/loader.py

@@ -162,6 +162,18 @@ def validate_config(checkpoint_path: str):
     return os.path.exists(checkpoint_path) and os.path.exists(
         os.path.join(checkpoint_path, 'config.json'))
 
+  @staticmethod
+  def validate_concept(checkpoint_path: str):
+    # check if downloaded concept exists in hf model
+    config_path = os.path.join(checkpoint_path, 'config.json')
+    with open(config_path, 'r') as f:
+      config = json.load(f)
+
+    labels = config.get('id2label', None)
+    if labels:
+      return True
+    return False
+
   @staticmethod
   def fetch_labels(checkpoint_path: str):
     # Fetch labels for classification, detection and segmentation models

tests/runners/test_model_run_locally-container.py

@@ -0,0 +1,116 @@
+import os
+import shutil
+import subprocess
+import sys
+
+import pytest
+from clarifai.runners.models.model_run_locally import ModelRunLocally
+
+CLARIFAI_USER_ID = os.environ["CLARIFAI_USER_ID"]
+CLARIFAI_PAT = os.environ["CLARIFAI_PAT"]
+
+
+@pytest.fixture
+def model_run_locally(dummy_models_path):
+  """
+  Fixture that instantiates the ModelRunLocally class
+  with the dummy model_path that already exists.
+  """
+  return ModelRunLocally(dummy_models_path)
+
+
+@pytest.fixture
+def hf_model_run_locally(dummy_hf_models_path):
+  """
+  Fixture that instantiates the ModelRunLocally class
+  with the dummy model_path that already exists.
+  """
+  return ModelRunLocally(dummy_hf_models_path)
+
+
+@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker not installed or not in PATH.")
+@pytest.mark.skipif(
+    sys.platform not in ["linux", "darwin"],
+    reason="Test only runs on Linux and macOS because base image only supports those platforms.")
+def test_docker_build_and_test_container(model_run_locally):
+  """
+  Test building a Docker image and running a container test using the dummy model.
+  This test will be skipped if Docker is not installed.
+  """
+
+  # Test if Docker is installed
+  assert model_run_locally.is_docker_installed(), "Docker not installed, skipping."
+
+  # Build or re-build the Docker image
+  model_run_locally.builder.create_dockerfile()
+  image_tag = model_run_locally._docker_hash()
+  image_name = f"{model_run_locally.config['model']['id']}:{image_tag}"
+
+  if not model_run_locally.docker_image_exists(image_name):
+    model_run_locally.build_docker_image(image_name=image_name)
+
+  # Run tests inside the container
+  try:
+    model_run_locally.test_model_container(
+        image_name=image_name,
+        container_name="test_clarifai_model_container",
+        env_vars={
+            'CLARIFAI_PAT': CLARIFAI_PAT,
+            'CLARIFAI_API_BASE': os.environ.get('CLARIFAI_API_BASE', 'https://api.clarifai.com')
+        })
+  except subprocess.CalledProcessError:
+    pytest.fail("Failed to test the model inside the docker container.")
+  finally:
+    # Clean up the container if it still exists
+    if model_run_locally.container_exists("test_clarifai_model_container"):
+      model_run_locally.stop_docker_container("test_clarifai_model_container")
+      model_run_locally.remove_docker_container("test_clarifai_model_container")
+
+    # Remove the image
+    model_run_locally.remove_docker_image(image_name)
+
+
+# Skip the test if Docker is not installed or if the platform is not Linux/macOS
+@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker not installed or not in PATH.")
+@pytest.mark.skipif(
+    sys.platform not in ["linux", "darwin"],
+    reason="Test only runs on Linux and macOS because base image only supports those platforms.")
+def test_hf_docker_build_and_test_container(hf_model_run_locally):
+  """
+  Test building a Docker image and running a container test using the dummy model.
+  This test will be skipped if Docker is not installed.
+  """
+
+  # Download the checkpoints for the model
+  hf_model_run_locally.builder.download_checkpoints()
+
+  # Test if Docker is installed
+  assert hf_model_run_locally.is_docker_installed(), "Docker not installed, skipping."
+
+  # Build or re-build the Docker image
+  hf_model_run_locally.builder.create_dockerfile()
+  image_tag = hf_model_run_locally._docker_hash()
+  image_name = f"{hf_model_run_locally.config['model']['id']}:{image_tag}"
+
+  if not hf_model_run_locally.docker_image_exists(image_name):
+    hf_model_run_locally.build_docker_image(image_name=image_name)
+
+  # Run tests inside the container
+  try:
+    hf_model_run_locally.test_model_container(
+        image_name=image_name,
+        container_name="test_clarifai_model_container",
+        env_vars={
+            'CLARIFAI_PAT': CLARIFAI_PAT,
+            'CLARIFAI_API_BASE': os.environ.get('CLARIFAI_API_BASE', 'https://api.clarifai.com')
+        })
+  except subprocess.CalledProcessError:
+    pytest.fail("Failed to test the model inside the docker container.")
+  finally:
+    # Clean up the container if it still exists
+    if hf_model_run_locally.container_exists("test_clarifai_model_container"):
+      hf_model_run_locally.stop_docker_container("test_clarifai_model_container")
+      hf_model_run_locally.remove_docker_container("test_clarifai_model_container")
+
+    # Remove the image
+    hf_model_run_locally.remove_docker_image(image_name)

tests/runners/test_model_run_locally.py

@@ -165,46 +165,6 @@ def test_test_model_success(model_run_locally):
     model_run_locally.clean_up()
 
 
-# @pytest.mark.skipif(shutil.which("docker") is None, reason="Docker not installed or not in PATH.")
-@pytest.mark.skip(reason="Will add later after new clarifai package is released")
-def test_docker_build_and_test_container(model_run_locally):
-  """
-  Test building a Docker image and running a container test using the dummy model.
-  This test will be skipped if Docker is not installed.
-  """
-
-  # Test if Docker is installed
-  assert model_run_locally.is_docker_installed(), "Docker not installed, skipping."
-
-  # Build or re-build the Docker image
-  model_run_locally.builder.create_dockerfile()
-  image_tag = model_run_locally._docker_hash()
-  image_name = f"{model_run_locally.config['model']['id']}:{image_tag}"
-
-  if not model_run_locally.docker_image_exists(image_name):
-    model_run_locally.build_docker_image(image_name=image_name)
-
-  # Run tests inside the container
-  try:
-    model_run_locally.test_model_container(
-        image_name=image_name,
-        container_name="test_clarifai_model_container",
-        env_vars={
-            'CLARIFAI_PAT': CLARIFAI_PAT,
-            'CLARIFAI_API_BASE': os.environ.get('CLARIFAI_API_BASE', 'https://api.clarifai.com')
-        })
-  except subprocess.CalledProcessError:
-    pytest.fail("Failed to test the model inside the docker container.")
-  finally:
-    # Clean up the container if it still exists
-    if model_run_locally.container_exists("test_clarifai_model_container"):
-      model_run_locally.stop_docker_container("test_clarifai_model_container")
-      model_run_locally.remove_docker_container("test_clarifai_model_container")
-
-    # Remove the image
-    model_run_locally.remove_docker_image(image_name)
-
-
 def test_hf_test_model_success(hf_model_run_locally):
   """
   Test that test_model succeeds with the dummy model.
@@ -225,46 +185,3 @@ def test_hf_test_model_success(hf_model_run_locally):
   finally:
     # Clean up
     hf_model_run_locally.clean_up()
-
-
-# @pytest.mark.skipif(shutil.which("docker") is None, reason="Docker not installed or not in PATH.")
-@pytest.mark.skip(reason="Will add later after new clarifai package is released")
-def test_hf_docker_build_and_test_container(hf_model_run_locally):
-  """
-  Test building a Docker image and running a container test using the dummy model.
-  This test will be skipped if Docker is not installed.
-  """
-
-  # Download the checkpoints for the model
-  hf_model_run_locally.builder.download_checkpoints()
-
-  # Test if Docker is installed
-  assert hf_model_run_locally.is_docker_installed(), "Docker not installed, skipping."
-
-  # Build or re-build the Docker image
-  hf_model_run_locally.builder.create_dockerfile()
-  image_tag = hf_model_run_locally._docker_hash()
-  image_name = f"{hf_model_run_locally.config['model']['id']}:{image_tag}"
-
-  if not hf_model_run_locally.docker_image_exists(image_name):
-    hf_model_run_locally.build_docker_image(image_name=image_name)
-
-  # Run tests inside the container
-  try:
-    hf_model_run_locally.test_model_container(
-        image_name=image_name,
-        container_name="test_clarifai_model_container",
-        env_vars={
-            'CLARIFAI_PAT': CLARIFAI_PAT,
-            'CLARIFAI_API_BASE': os.environ.get('CLARIFAI_API_BASE', 'https://api.clarifai.com')
-        })
-  except subprocess.CalledProcessError:
-    pytest.fail("Failed to test the model inside the docker container.")
-  finally:
-    # Clean up the container if it still exists
-    if hf_model_run_locally.container_exists("test_clarifai_model_container"):
-      hf_model_run_locally.stop_docker_container("test_clarifai_model_container")
-      hf_model_run_locally.remove_docker_container("test_clarifai_model_container")
-
-    # Remove the image
-    hf_model_run_locally.remove_docker_image(image_name)