diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..db60049 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.github +.pytest_cache +.ruff_cache +.venv* +build +chroma_data +dist +tmp +**/__pycache__ +*.py[cod] +.env diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..00d0253 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +VIDXP_IMAGE=ghcr.io/grayhatdevelopers/vidxp:latest +VIDXP_PORT=8501 +VIDXP_DEVICE=cpu diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1c55fbc --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,35 @@ +version: 2 +updates: + - package-ecosystem: pip + directories: + - "/" + - "/src/vidxp/requirements" + - "/src/vidxp/capabilities/dialogue" + - "/src/vidxp/capabilities/scene" + - "/src/vidxp/capabilities/actor" + - "/src/vidxp/benchmarks" + - "/utils" + schedule: + interval: cron + cronjob: "0 9 1,15 * *" + timezone: UTC + groups: + compatible-updates: + patterns: + - "*" + update-types: + - minor + - patch + open-pull-requests-limit: 2 + + - package-ecosystem: github-actions + directory: / + schedule: + interval: cron + cronjob: "0 9 1,15 * *" + timezone: UTC + groups: + github-actions: + patterns: + - "*" + open-pull-requests-limit: 2 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..b1d884b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,13 @@ +## Summary + +- Describe the user-facing outcome. + +## Validation + +- List the commands or checks run. + +## Changelog + +- [ ] Added `changes/..md` +- [ ] This change is internal-only; explain why a maintainer should apply the + `skip-changelog` label: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..41c3545 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,149 @@ +name: CI + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + - unlabeled + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + cache: pip + + - name: Install repository tooling + run: | + python -m pip install --upgrade pip + python -m pip install -r utils/build-requirements.txt + + - name: Require a changelog fragment + if: >- + github.event_name == 'pull_request' && + !contains(github.event.pull_request.labels.*.name, 'skip-changelog') + run: towncrier check --compare-with "${{ github.event.pull_request.base.sha }}" + + - name: Determine validation scope + id: scope + shell: bash + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + changed_files="$(git diff --name-only "$BASE_SHA" HEAD)" + if grep -Eq '^(\.github/workflows/ci\.yml$|src/|tests/|utils/|LICENSE$|MANIFEST\.in$|pyproject\.toml$)' <<< "$changed_files"; then + run_suite=true + else + run_suite=false + fi + if grep -Eq '^(\.dockerignore$|\.github/workflows/ci\.yml$|Dockerfile$|compose\.yaml$|pyproject\.toml$|src/vidxp/(requirements/.*\.txt|capabilities/[^/]+/requirements\.txt)$)' <<< "$changed_files"; then + run_container=true + else + run_container=false + fi + echo "run_suite=$run_suite" >> "$GITHUB_OUTPUT" + echo "run_container=$run_container" >> "$GITHUB_OUTPUT" + + - name: Install test surface + if: steps.scope.outputs.run_suite == 'true' + run: python -m pip install ".[scene,frontend,benchmarks]" + + - name: Lint + if: steps.scope.outputs.run_suite == 'true' + run: ruff check src tests + + - name: Test + if: steps.scope.outputs.run_suite == 'true' + run: python -m unittest discover -s tests -q + env: + PYTHONPATH: src + + - name: Build wheel and source distribution + if: steps.scope.outputs.run_suite == 'true' + run: python -m build + + - name: Verify minimal wheel + if: steps.scope.outputs.run_suite == 'true' + shell: bash + run: | + python -m venv .wheel-smoke + .wheel-smoke/bin/python -m pip install --upgrade pip + .wheel-smoke/bin/python -m pip install dist/*.whl + .wheel-smoke/bin/python -m pip check + .wheel-smoke/bin/python - <<'PY' + from importlib.util import find_spec + from subprocess import check_output + + import vidxp + from vidxp.capabilities.registry import capability_names + + assert capability_names() == ("dialogue", "scene", "actor") + help_text = check_output( + [".wheel-smoke/bin/vidxp", "--help"], + text=True, + ) + assert "benchmark" not in help_text + for module in ( + "chromadb", + "clip", + "cv2", + "face_recognition", + "sentence_transformers", + "srt", + "streamlit", + "torch", + "whisperx", + ): + assert find_spec(module) is None, module + PY + + - name: Validate Compose configuration + if: steps.scope.outputs.run_container == 'true' + run: docker compose config --quiet + + - name: Build container + if: steps.scope.outputs.run_container == 'true' + run: docker build --tag vidxp:ci . + + - name: Smoke container + if: steps.scope.outputs.run_container == 'true' + shell: bash + run: | + docker run --detach --name vidxp-ci \ + --publish 127.0.0.1:8501:8501 vidxp:ci + trap 'docker rm --force vidxp-ci >/dev/null 2>&1 || true' EXIT + + docker exec vidxp-ci vidxp --version + docker exec vidxp-ci python -c \ + 'import clip, face_recognition, whisperx' + docker exec vidxp-ci sh -c \ + 'test ! -d "$HOME/.cache/huggingface" && test ! -d "$HOME/.cache/clip"' + + for attempt in {1..30}; do + if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then + exit 0 + fi + if [[ "$(docker inspect --format='{{.State.Running}}' vidxp-ci)" != "true" ]]; then + break + fi + sleep 2 + done + + docker logs vidxp-ci + exit 1 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..8eb1ef8 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,37 @@ +name: CodeQL + +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/codeql.yml" + - "src/**/*.py" + - "utils/**/*.py" + push: + branches: + - main + paths: + - ".github/workflows/codeql.yml" + - "src/**/*.py" + - "utils/**/*.py" + schedule: + - cron: "17 4 * * 6" + +concurrency: + group: codeql-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write + +jobs: + analyze: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: github/codeql-action/init@v4 + with: + languages: python + - uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/release-to-pypi.yml b/.github/workflows/release-to-pypi.yml index c4ef7da..e2d9b08 100644 --- a/.github/workflows/release-to-pypi.yml +++ b/.github/workflows/release-to-pypi.yml @@ -1,29 +1,26 @@ -name: Release (release → PyPI) +name: Release (main → PyPI) on: - push: - branches: - - release - paths: - - "src/**" - - "pyproject.toml" - - "README.md" - - "LICENSE" - - "MANIFEST.in" - - "utils/build_package.sh" - - "utils/build-requirements.txt" - - "utils/fix_readme_links.py" + workflow_dispatch: permissions: - contents: write - id-token: write - pull-requests: write + contents: read + +concurrency: + group: pypi-main + cancel-in-progress: false jobs: release: + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest - environment: - name: pypi + permissions: + contents: write + outputs: + commit_sha: ${{ steps.release.outputs.commit_sha }} + released: ${{ steps.release.outputs.released }} + tag: ${{ steps.release.outputs.tag }} + version: ${{ steps.release.outputs.version }} steps: - uses: actions/checkout@v7 @@ -38,18 +35,145 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install build "python-semantic-release~=10.6.0" - pip install -r utils/build-requirements.txt + python -m pip install -r utils/build-requirements.txt - name: Create stable release id: release env: + BUILD_CHANGELOG: "1" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: semantic-release version --no-changelog + + - name: Smoke distribution + if: steps.release.outputs.released == 'true' + shell: bash run: | - semantic-release version --commit --tag --push + python -m venv .release-smoke + .release-smoke/bin/python -m pip install dist/*.whl + .release-smoke/bin/python -m pip check + .release-smoke/bin/vidxp --version - - name: Publish to PyPI + - name: Upload GitHub release assets + if: steps.release.outputs.released == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: semantic-release publish --tag "${{ steps.release.outputs.tag }}" + + - name: Preserve distribution if: steps.release.outputs.released == 'true' + uses: actions/upload-artifact@v4 + with: + name: pypi-distribution + path: dist/ + if-no-files-found: error + retention-days: 7 + + publish-pypi: + if: needs.release.outputs.released == 'true' + needs: release + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + + steps: + - uses: actions/download-artifact@v5 + with: + name: pypi-distribution + path: dist/ + + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ + + publish-container: + if: >- + github.ref == 'refs/heads/main' && + needs.release.outputs.released == 'true' + needs: release + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.release.outputs.commit_sha }} + + - uses: docker/setup-buildx-action@v3 + + - name: Generate container metadata + id: metadata + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern={{version}},value=${{ needs.release.outputs.tag }} + type=semver,pattern={{major}}.{{minor}},value=${{ needs.release.outputs.tag }} + type=raw,value=latest + labels: | + org.opencontainers.image.title=VidXP + org.opencontainers.image.description=Local-first video indexing and search + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ needs.release.outputs.commit_sha }} + org.opencontainers.image.version=${{ needs.release.outputs.version }} + org.opencontainers.image.licenses=MIT + + - name: Build stable container + uses: docker/build-push-action@v6 + with: + context: . + load: true + platforms: linux/amd64 + push: false + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + + - name: Smoke stable container + shell: bash + run: | + image="ghcr.io/${{ github.repository }}:${{ needs.release.outputs.version }}" + docker run --detach --name vidxp-release \ + --publish 127.0.0.1:8501:8501 "$image" + trap 'docker rm --force vidxp-release >/dev/null 2>&1 || true' EXIT + + docker exec vidxp-release vidxp --version + docker exec vidxp-release python -c \ + 'import clip, face_recognition, whisperx' + docker exec vidxp-release sh -c \ + 'test ! -d "$HOME/.cache/huggingface" && test ! -d "$HOME/.cache/clip"' + + for attempt in {1..30}; do + if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then + exit 0 + fi + if [[ "$(docker inspect --format='{{.State.Running}}' vidxp-release)" != "true" ]]; then + break + fi + sleep 2 + done + + docker logs vidxp-release + exit 1 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish stable container + run: docker push --all-tags "ghcr.io/${{ github.repository }}" + + - name: Verify public container access + shell: bash + run: | + docker logout ghcr.io + image="ghcr.io/${{ github.repository }}:${{ needs.release.outputs.version }}" + if ! docker buildx imagetools inspect "$image"; then + echo "::error::Make the GHCR package public, then rerun this failed job." + exit 1 + fi diff --git a/.github/workflows/release-to-test-pypi.yml b/.github/workflows/release-to-test-pypi.yml index fdd4f5a..a233dd9 100644 --- a/.github/workflows/release-to-test-pypi.yml +++ b/.github/workflows/release-to-test-pypi.yml @@ -8,20 +8,27 @@ on: - "src/**" - "pyproject.toml" - "README.md" + - "CHANGELOG.md" + - "changes/**" - "LICENSE" - "MANIFEST.in" - - "utils/build_package.sh" - - "utils/build-requirements.txt" - - "utils/fix_readme_links.py" + - "utils/**" + - ".github/workflows/release-to-test-pypi.yml" permissions: - contents: write - id-token: write + contents: read + +concurrency: + group: testpypi-${{ github.ref }} + cancel-in-progress: false jobs: prerelease: runs-on: ubuntu-latest - environment: testpypi + permissions: + contents: write + outputs: + released: ${{ steps.release.outputs.released }} steps: - uses: actions/checkout@v7 @@ -36,18 +43,53 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install build "python-semantic-release~=10.6.0" - pip install -r utils/build-requirements.txt + python -m pip install -r utils/build-requirements.txt - name: Determine and tag prerelease version id: release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: semantic-release version --as-prerelease --prerelease-token b --no-changelog + + - name: Smoke distribution + if: steps.release.outputs.released == 'true' + shell: bash run: | - semantic-release version --commit --tag --push + python -m venv .release-smoke + .release-smoke/bin/python -m pip install dist/*.whl + .release-smoke/bin/python -m pip check + .release-smoke/bin/vidxp --version - - name: Publish to TestPyPI + - name: Upload GitHub release assets + if: steps.release.outputs.released == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: semantic-release publish --tag "${{ steps.release.outputs.tag }}" + + - name: Preserve distribution if: steps.release.outputs.released == 'true' + uses: actions/upload-artifact@v4 + with: + name: testpypi-distribution + path: dist/ + if-no-files-found: error + retention-days: 7 + + publish-testpypi: + if: needs.prerelease.outputs.released == 'true' + needs: prerelease + runs-on: ubuntu-latest + environment: testpypi + permissions: + id-token: write + + steps: + - uses: actions/download-artifact@v5 + with: + name: testpypi-distribution + path: dist/ + + - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..268772c --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,26 @@ +name: Security + +on: + pull_request: + paths: + - ".github/dependabot.yml" + - ".github/workflows/**" + - "pyproject.toml" + - "src/**/*.txt" + - "utils/build-requirements.txt" + +concurrency: + group: security-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/dependency-review-action@v5 + with: + fail-on-severity: high diff --git a/CHANGELOG.md b/CHANGELOG.md index abadeef..0162af7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # CHANGELOG - + + +## v0.1.0 (2026-07-27) + ## v0.1.0-b.3 (2026-07-27) diff --git a/Dockerfile b/Dockerfile index 4e64364..e74c2f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.10-slim +# syntax=docker/dockerfile:1 + +FROM python:3.11-slim-bookworm AS builder WORKDIR /app @@ -7,27 +9,85 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ git \ pkg-config \ - ffmpeg \ - libglib2.0-0 \ - libgl1 \ - libsm6 \ libopenblas-dev \ liblapack-dev \ libjpeg62-turbo-dev \ libpng-dev \ - libxext6 \ - libxrender1 \ && rm -rf /var/lib/apt/lists/* -ENV PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 +RUN python -m venv /opt/vidxp + +ENV PATH="/opt/vidxp/bin:${PATH}" \ + PIP_DISABLE_PIP_VERSION_CHECK=1 COPY pyproject.toml README.md LICENSE MANIFEST.in ./ +COPY src/vidxp/requirements ./src/vidxp/requirements +COPY src/vidxp/capabilities/dialogue/requirements.txt ./src/vidxp/capabilities/dialogue/requirements.txt +COPY src/vidxp/capabilities/scene/requirements.txt ./src/vidxp/capabilities/scene/requirements.txt +COPY src/vidxp/capabilities/actor/requirements.txt ./src/vidxp/capabilities/actor/requirements.txt + +ARG PYTORCH_INDEX_URL="https://download.pytorch.org/whl/cpu" + +RUN --mount=type=cache,target=/root/.cache/pip \ + python -m pip install --upgrade pip setuptools wheel \ + && python -m pip install \ + --extra-index-url "${PYTORCH_INDEX_URL}" \ + -r src/vidxp/requirements/storage.txt \ + -r src/vidxp/requirements/frontend.txt \ + -r src/vidxp/capabilities/dialogue/requirements.txt \ + -r src/vidxp/capabilities/scene/requirements.txt \ + -r src/vidxp/capabilities/actor/requirements.txt + COPY src ./src -RUN python -m pip install --upgrade pip setuptools wheel \ - && pip install ".[frontend]" +RUN --mount=type=cache,target=/root/.cache/pip \ + python -m pip install . \ + && python -m pip check + +FROM python:3.11-slim-bookworm AS runtime + +LABEL org.opencontainers.image.title="VidXP" \ + org.opencontainers.image.description="Local-first video indexing and search" \ + org.opencontainers.image.source="https://github.com/grayhatdevelopers/vidxp" \ + org.opencontainers.image.licenses="MIT" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + libglib2.0-0 \ + libgl1 \ + libgomp1 \ + libjpeg62-turbo \ + liblapack3 \ + libopenblas0 \ + libpng16-16 \ + libsm6 \ + libxext6 \ + libxrender1 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system vidxp \ + && useradd --system --gid vidxp --home-dir /var/lib/vidxp vidxp \ + && mkdir -p /var/lib/vidxp \ + && chown vidxp:vidxp /var/lib/vidxp + +COPY --from=builder /opt/vidxp /opt/vidxp + +ENV PATH="/opt/vidxp/bin:${PATH}" \ + HOME="/var/lib/vidxp" \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + STREAMLIT_BROWSER_GATHER_USAGE_STATS=false \ + STREAMLIT_SERVER_HEADLESS=true \ + VIDXP_CONFIG_FILE="/var/lib/vidxp/config/repositories.json" \ + VIDXP_INDEX_DIR="/var/lib/vidxp/index" + +USER vidxp +WORKDIR /var/lib/vidxp + +VOLUME ["/var/lib/vidxp"] EXPOSE 8501 -CMD ["vidxp-ui", "--server.address=0.0.0.0", "--server.port=8501"] +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8501/_stcore/health', timeout=3)"] + +CMD ["vidxp", "ui", "--host", "0.0.0.0", "--port", "8501"] diff --git a/INSTALLATION_GUIDE.md b/INSTALLATION_GUIDE.md index 182d5e9..fc5c747 100644 --- a/INSTALLATION_GUIDE.md +++ b/INSTALLATION_GUIDE.md @@ -7,8 +7,8 @@ installation, model preparation, and common first-run issues. ## Prerequisites - Python 3.10 through 3.13 -- FFmpeg available on `PATH` -- CMake and a C/C++ build toolchain for `dlib` +- FFmpeg available on `PATH` when installing `dialogue` +- CMake and a C/C++ build toolchain when installing `actor` The official `dlib` package is distributed through PyPI as source. Installing VidXP therefore compiles it locally unless a compatible build is already cached @@ -43,19 +43,29 @@ python -m pip install --upgrade pip ## Install from PyPI -Install the command-line package: +Install the lightweight command and application layer: ```bash python -m pip install vidxp ``` -Install the command-line package and Streamlit interface: +Add only the indexing capabilities you need: ```bash -python -m pip install "vidxp[frontend]" +python -m pip install "vidxp[dialogue]" +python -m pip install "vidxp[scene,actor]" +python -m pip install "vidxp[all]" ``` -`frontend` is an optional dependency group. The package name remains `vidxp`. +`all` contains all runtime capabilities. It does not include the `frontend` or +`benchmarks` extras: + +```bash +python -m pip install "vidxp[all,frontend]" +``` + +The `benchmarks` extra contains evaluation tooling only. Combine it with the +capability being evaluated, for example `vidxp[scene,benchmarks]`. ## Install from source @@ -65,16 +75,44 @@ From the repository root: python -m pip install . ``` -Include the Streamlit interface: +Include all capabilities and the Streamlit interface: ```bash -python -m pip install ".[frontend]" +python -m pip install ".[all,frontend]" ``` Use an editable installation while developing: ```bash -python -m pip install -e ".[frontend,benchmarks]" +python -m pip install -e ".[all,frontend,benchmarks]" +``` + +## Run the container + +Stable container images include every runtime capability and the browser +interface, but not downloaded model weights. Docker Compose keeps indexes, +repository configuration, and model caches in the `vidxp-data` volume: + +```bash +docker compose pull +docker compose run --rm vidxp vidxp prepare +docker compose up +``` + +The interface is available at `http://localhost:8501`. Set `VIDXP_PORT` in a +local `.env` file to publish a different host port. Set `VIDXP_IMAGE` to select +an immutable release instead of `latest`: + +```dotenv +VIDXP_IMAGE=ghcr.io/grayhatdevelopers/vidxp:0.2.0 +VIDXP_PORT=8502 +VIDXP_DEVICE=cpu +``` + +Run the image without Compose when persistent storage is not required: + +```bash +docker run --rm -p 8501:8501 ghcr.io/grayhatdevelopers/vidxp:latest ``` ## Verify the installation @@ -85,21 +123,23 @@ Display the installed package version: vidxp --version ``` -Check FFmpeg and the Python dependencies needed by all indexing capabilities: +Check the Python and system dependencies needed by all indexing capabilities: ```bash vidxp doctor ``` `vidxp doctor` imports the selected dependencies but does not download model -weights. Restrict the check when diagnosing one capability: +weights. A base-only install therefore reports which capability extras are +missing. Restrict the check when diagnosing one capability: ```bash vidxp doctor --modalities scene ``` -If the frontend extra was installed, start the interface with `vidxp-ui`. It -runs until stopped with `Ctrl+C`. +If the frontend extra was installed, start the interface with `vidxp ui`. It +uses the active repository configured by `vidxp repositories use` and runs +until stopped with `Ctrl+C`. ## Prepare models @@ -123,7 +163,7 @@ WhisperX selects its alignment model after detecting the video's language. Cache a known language explicitly when required: ```bash -vidxp prepare --language en +vidxp prepare --option dialogue.alignment_language=en ``` SentenceTransformer and WhisperX use the Hugging Face cache; CLIP uses its own @@ -139,7 +179,7 @@ If `vidxp prepare` was not run first, the initial indexing command downloads any missing models before processing the video: ```bash -vidxp videoindex samplevideo.mp4 +vidxp index create samplevideo.mp4 ``` Keep the terminal and internet connection open until VidXP reports that the @@ -167,11 +207,12 @@ directory to `PATH`, then rerun `vidxp doctor`. ### A search says the index is not ready Wait for the active indexing command to finish. If the previous process ended -or failed, run `vidxp videoindex` again to rebuild the incomplete local index. +or failed, run `vidxp index create` again to rebuild the incomplete local +index. ### The first indexing run appears slow Check the terminal for model-download or indexing progress. Model preparation, transcription, scene analysis, and actor detection are separate stages. Use `vidxp prepare` before indexing or select fewer capabilities with -`--modalities`. +one or more `--modality` options. diff --git a/README.md b/README.md index 21caec9..d151fda 100644 --- a/README.md +++ b/README.md @@ -14,45 +14,60 @@

VidXP is a local-first video indexing and search engine distributed as a Python package. -
You can use it:

+ +
+
You can use it: - -

- - download - + Dialogue search · Scene search · Actor grouping · CLI · Browser UI · Python API

+ + PyPI version + + + Supported Python versions + + + CI status + + + MIT license + - discord + Discord

- ## Why VidXP Finding one moment in a video should not require scrubbing through the entire timeline. VidXP builds a searchable index from three kinds of evidence: -- **Dialogue:** semantic search over timestamped WhisperX transcripts. -- **Scenes:** text-to-frame search using CLIP. -- **Actors:** groups similar detected faces and exports a highlighted video for - a selected cluster. +- **Dialogue:** semantic search over timestamped transcripts. +- **Scenes:** text-to-frame search. +- **Actors:** groups similar detected faces and exports a highlighted video for a selected cluster. + +After the required model weights are available, video processing and search run completely locally, for your privacy and security. + +Some ideas on how to use VidXP: +- Use it as way to find your favorite relatives in a huge folder of wedding videos (been there, done that) +- Use it in your application, allow users to search videos (an idea: use it alongside a video-editing application) +- Use it as an "understanding" layer so your LLM / agent can understand videos + +[![Video Screenshot](./docs/images/video-screenshot.jpeg)](https://www.linkedin.com/feed/update/urn:li:activity:7343569473720725505/) -After the required model weights are available, video processing and search run -locally. VidXP also saves index state so an incomplete run is not mistaken for a -searchable result. ## Current capabilities @@ -70,31 +85,20 @@ VidXP supports Python 3.10 through 3.13 and requires FFmpeg. See the [installation guide](INSTALLATION_GUIDE.md) for the `dlib` compiler requirements, source installation, model preparation, and troubleshooting. -Create and activate a virtual environment: +Install the command line and browser interface with +[pipx](https://packaging.python.org/en/latest/guides/installing-stand-alone-command-line-tools/). +The command is available on your `PATH` while VidXP and its dependencies remain +isolated: ```bash -python -m venv venv +pipx install "vidxp[all,frontend]" ``` -```bash -# Windows -venv\Scripts\activate - -# macOS/Linux -source venv/bin/activate -``` - -Install the command-line package: - -```bash -python -m pip install vidxp -``` - -Include the browser interface: - -```bash -python -m pip install "vidxp[frontend]" -``` +Install only the capabilities you need with a smaller selection such as +`pipx install "vidxp[scene,frontend]"` or +`pipx install "vidxp[dialogue,scene]"`. To import VidXP from another Python +project, install it into that project's environment instead; the +[installation guide](INSTALLATION_GUIDE.md) covers that path. Confirm the installed package and its runtime dependencies: @@ -115,40 +119,62 @@ vidxp prepare Build an index containing dialogue, scene, and actor information: ```bash -vidxp videoindex samplevideo.mp4 +vidxp index create samplevideo.mp4 ``` Search the completed index: ```bash -vidxp dialogue "the bread just came out of the oven" -vidxp scene "a yellow taxi on a city street" -vidxp actor 1 samplevideo.mp4 +vidxp search dialogue "the bread just came out of the oven" +vidxp search scene "a yellow taxi on a city street" --top-k 5 +vidxp actors list +vidxp actors render 1 samplevideo.mp4 ``` Index only selected capabilities or sample fewer visual frames: ```bash -vidxp videoindex samplevideo.mp4 --modalities scene --frame-stride 5 +vidxp index create samplevideo.mp4 --modality scene --frame-stride 5 ``` -`--modalities` accepts any combination of `dialogue`, `scene`, and `actor`. +Repeat `--modality` to combine `dialogue`, `scene`, and `actor`. Run `vidxp --help` or any command followed by `--help` for the complete command reference. +Use named repositories to keep index locations and devices centrally +configured: + +```bash +vidxp repositories add team --index-dir ./indexes/team --device cuda --use +vidxp repositories list +``` + ## Browser interface Install the `frontend` extra and start: ```bash -vidxp-ui +vidxp ui ``` -The command starts a local Streamlit server and remains active until stopped. +The command uses the active named repository, starts a local Streamlit server, +and remains active until stopped. The interface can upload a video, start or cancel indexing, restore saved progress after a page reload, and search the capabilities available in the completed index. +## Container + +Stable releases are available from GitHub Container Registry. Start the local +interface with persistent index and model storage by running: + +```bash +docker compose up +``` + +See the [installation guide](INSTALLATION_GUIDE.md#run-the-container) for model +preparation, configuration, and direct `docker run` usage. + ## Use VidXP as a Python package The programmatic API supports isolated multi-video runs, supplied timestamped @@ -157,7 +183,7 @@ transcripts, resumable per-video checkpoints, and metadata-rich top-k results. ```python from vidxp.core import IndexConfig, VideoSource from vidxp.core.runner import run_index -from vidxp.core.search import search_scene +from vidxp.capabilities.scene.operations import search_scene config = IndexConfig( dataset="my-library", @@ -189,24 +215,9 @@ documents configuration, stored metadata, result fields, and run layout. --- - -## Current scope - -- The standard CLI and browser interface manage one local searchable video - index at a time. The Python layer supports isolated multi-video runs. -- Actor clusters represent visually similar detected faces; VidXP does not - automatically know or assign a person's name. -- Model weights are cached separately from the Python package and require - additional disk space. -- CPU is the current default. Visual indexing processes every frame unless - `--frame-stride` is configured. -- Search quality depends on the selected models, video domain, speech quality, - and frame sampling. - ## Roadmap -VidXP is an evolving beta. The roadmap extends the current engine rather than -replacing its working search paths. +VidXP is an evolving beta. We'd love to hear your feedback and where you'd like to see the project go. | Area | Current foundation | Direction | |---|---|---| @@ -218,8 +229,6 @@ replacing its working search paths. | Product experience | CLI and browser indexing/search | Clearer progress, result navigation, recovery, and long-running job controls | | Evaluation | DiDeMo and HiREST baselines | Combined and component benchmarks, beginning with a LongVALE pilot | -Roadmap items describe intended direction, not a release guarantee. - ## Models and local data | Capability | Model | @@ -237,7 +246,7 @@ caches normally live outside this directory and outside the virtual environment. - [Installation and troubleshooting](INSTALLATION_GUIDE.md) - [Benchmarking status and results](docs/benchmarking/README.md) -- [Contribution guidelines](docs/CONTRIBUTING.md) +- [Adding a capability](docs/adding-a-capability.md) - [Changelog](CHANGELOG.md) - [Issue tracker](https://github.com/grayhatdevelopers/vidxp/issues) - [MIT license](LICENSE) @@ -249,10 +258,10 @@ See [CONTRIBUTING.md](./docs/CONTRIBUTING.md) for guidelines, maintainers, and h ## Credits -Built by Grayhat Developers PVT Ltd in 2025. Maintained by the community. +Built by Grayhat Developers PVT Ltd. 2026. Maintained by the community. Email: info@grayhat.studio - \ No newline at end of file + diff --git a/changes/11.feature.md b/changes/11.feature.md new file mode 100644 index 0000000..0b418ac --- /dev/null +++ b/changes/11.feature.md @@ -0,0 +1 @@ +Add a structured command-line interface and reusable application service with named repository configuration. diff --git a/changes/12.feature.md b/changes/12.feature.md new file mode 100644 index 0000000..9023f87 --- /dev/null +++ b/changes/12.feature.md @@ -0,0 +1 @@ +Add registry-driven capabilities with selective installation extras, an explicit benchmark add-on, and stable container images. diff --git a/changes/README.md b/changes/README.md new file mode 100644 index 0000000..1cc4a5b --- /dev/null +++ b/changes/README.md @@ -0,0 +1,28 @@ +# Changelog fragments + +Every pull request with a user-visible change must add one short fragment here. +Use the pull request number and the most specific type: + +```text +changes/..md +``` + +Supported types are `breaking`, `feature`, `bugfix`, `deprecation`, `docs`, and +`security`. Write one sentence for users in the imperative voice and do not add +a heading or the version number. + +Example: + +```text +changes/123.feature.md +``` + +```markdown +Add named repositories for selecting shared index locations and devices. +``` + +Purely internal changes may omit a fragment only when a maintainer applies the +`skip-changelog` label and the pull request explains why. + +Towncrier collects and removes fragments when a stable release is created. +Do not edit `CHANGELOG.md` directly. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..8085ce5 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,13 @@ +services: + vidxp: + image: ${VIDXP_IMAGE:-ghcr.io/grayhatdevelopers/vidxp:latest} + ports: + - "${VIDXP_PORT:-8501}:8501" + environment: + VIDXP_DEVICE: ${VIDXP_DEVICE:-cpu} + volumes: + - vidxp-data:/var/lib/vidxp + init: true + +volumes: + vidxp-data: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 899c763..df703a5 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -7,8 +7,11 @@ Thanks for contributing to VidXP (Video eXPlain). | File / path | Role | |-------------|------| | `src/vidxp/cli.py` | Typer commands and installed `vidxp` entry point | -| `src/vidxp/frontend.py` | Streamlit interface and installed `vidxp-ui` entry point | -| `src/vidxp/core/` | Indexing, retrieval, storage, models, run state, and shared contracts | +| `src/vidxp/application.py` | Reusable application boundary for CLI and future adapters | +| `src/vidxp/capabilities/` | Capability registry, schemas, operations, dependencies, and optional CLI modules | +| `src/vidxp/repositories.py` | Persistent named local-index configuration | +| `src/vidxp/frontend.py` | Streamlit interface launched by `vidxp ui` | +| `src/vidxp/core/` | Capability-neutral storage, media, run state, and execution contracts | | `src/vidxp/benchmarks/` | Benchmark-specific loaders, prediction adapters, and evaluator calls | | `pyproject.toml` | Package metadata and Python dependencies | | `docs/` | Installation-linked guidance, benchmark research, and contribution notes | @@ -17,7 +20,7 @@ Thanks for contributing to VidXP (Video eXPlain). | Model caches | Managed by WhisperX, SentenceTransformer, and CLIP outside the repository | The full local CLI/UI index uses up to three collections: -`voiceEmbeddings`, `sceneEmbeddings`, and `actorCollection`. Runs containing +`dialogue`, `scene`, and `actor`. Runs containing selected capabilities create only the collections they need. ## Setup @@ -29,7 +32,7 @@ package metadata directly. python -m venv venv # Windows: venv\Scripts\activate # macOS/Linux: source venv/bin/activate -python -m pip install -e ".[frontend,benchmarks]" +python -m pip install -e ".[all,frontend,benchmarks]" ``` Verify the environment: @@ -40,7 +43,7 @@ vidxp doctor ``` CLI: `vidxp --help` -UI: `vidxp-ui` +UI: `vidxp ui` Models default to CPU and download into their libraries' standard caches on first use. If a model identifier changes, update the setup documentation and @@ -48,14 +51,17 @@ record the exact identifier in benchmark results. ## Where to put work -- Indexing, retrieval, storage, metadata, and face clustering: +- Capability-specific indexing, retrieval, models, schemas, and dependencies: + the matching folder under `src/vidxp/capabilities/`. +- Shared storage, media handling, run state, and execution mechanics: `src/vidxp/core/`. +- Transport-neutral application operations: `src/vidxp/application.py`. - Command-line behavior: `src/vidxp/cli.py`. - Upload and search UX: `src/vidxp/frontend.py`; keep product logic in the - shared core. + shared application and core modules. - Official benchmark formats and evaluator calls: `src/vidxp/benchmarks/`. -- New dependencies: `pyproject.toml`, with the reason stated in the pull - request. +- Capability dependencies: that capability's `requirements.txt`; wire a new + install extra into `pyproject.toml`. - Product direction: the roadmap in the main [README](../README.md). Prefer small, focused pull requests. If you change how embeddings or metadata @@ -85,7 +91,34 @@ python -m unittest discover -s tests - Note any new env vars, model downloads, or breaking index format changes. - Link a related issue when there is one. +### Changelog fragments + +Add one `changes/..md` file for every user-visible pull +request. Use one of these types: + +- `breaking` for incompatible behavior or API changes. +- `feature` for new behavior. +- `bugfix` for corrected behavior. +- `deprecation` for behavior scheduled for removal. +- `docs` for user-facing documentation improvements. +- `security` for security fixes or hardening users should know about. + +The fragment should be one sentence written for users. Do not include a heading, +version number, commit message, or implementation details. See +[`changes/README.md`](../changes/README.md) for an example. + +Internal-only maintenance does not need an empty fragment. Explain the reason in +the pull request and ask a maintainer to apply the `skip-changelog` label. CI +requires either a fragment or that label. + +Towncrier collects the pending fragments into `CHANGELOG.md` and removes them +when a stable release is made. Do not edit the changelog or package version in a +feature pull request. + ## Questions -Open an issue before large refactors or new modalities so scope stays aligned -with the roadmap. +Follow [Adding a capability](adding-a-capability.md) for the complete extension +contract. Open an issue before large cross-capability refactors so scope stays +aligned with the roadmap. + +Maintainers should follow the [release process](releasing.md). diff --git a/docs/adding-a-capability.md b/docs/adding-a-capability.md new file mode 100644 index 0000000..d12ebf6 --- /dev/null +++ b/docs/adding-a-capability.md @@ -0,0 +1,95 @@ +# Adding a capability + +A capability is a self-contained VidXP feature with its own operations, +schemas, runtime dependencies, and optional indexing or CLI integration. +Chroma collections are storage details; do not call capability packages +“collections.” + +## Required shape + +Create `src/vidxp/capabilities//` with only the files the feature needs: + +```text +/ +├── __init__.py +├── definition.py +├── config.py # Pydantic settings owned by the capability +├── operations.py +├── schemas.py # omit when shared schemas are sufficient +├── requirements.txt +├── indexing.py # only for an indexable capability +└── cli.py # only for specialized commands +``` + +`definition.py` exports one frozen, Pydantic-validated +`CapabilityDefinition` named `DEFINITION`. +Register it explicitly in `src/vidxp/capabilities/registry.py`. The registry is +the only central file that should change for ordinary runtime registration. + +## Contracts + +Every public operation declares: + +- a Pydantic input model; +- a Pydantic output model; +- a transport-neutral handler; +- whether it requires an active index. + +Handlers receive `CapabilityContext`. Use `context.require_config()` only for +operations that declare `requires_index=True`. Do not import Typer, Streamlit, +FastAPI, or an MCP implementation into operation modules. + +An indexable capability also declares its collection names, indexing handler, +and index stage. The generic runner groups capabilities that reference the same +indexing handler, allowing related capabilities to share work without adding a +name switch to the runner. A capability joining a shared decoder also supplies +its own prepare/process/finalize processor through its definition; the shared +handler must not import individual capabilities. Capability-specific settings belong in +`IndexConfig.capability_options` and are read with +`config.options_for("")`. Validate them through the capability's +Pydantic settings model; do not add feature-specific fields to `IndexConfig` +or the runner. + +Operation-only capabilities leave the indexing fields unset. They do not need +dummy collections or index handlers. + +## Dependencies and installation + +Put direct runtime requirements in the capability's `requirements.txt`. +This file is the only declaration of Python package dependencies: setuptools +uses it to build the extra, and runtime dependency checks read the same packaged +file. Declare a `RuntimeCheck` only for a non-Python environment prerequisite +such as an executable. Expose the requirements file as an optional dependency +in `pyproject.toml`: + +```toml +[tool.setuptools.dynamic.optional-dependencies] +example = { file = ["src/vidxp/capabilities/example/requirements.txt"] } +``` + +Add runtime capabilities to the `all` file list. Do not add benchmark, +frontend, or development dependencies to `all`. Imports of optional libraries +must stay inside the functions that use them so `import vidxp` and +`vidxp --help` work in a base-only installation. + +## CLI integration + +Generic commands obtain capability names and operations from the registry. +Only add `cli.py` when the capability needs specialized human interaction that +does not fit a generic command. Expose it through `cli_name` and `cli_factory` +on the definition; keep business logic in operations. + +## Tests + +Add focused tests under `tests/`. At minimum, cover: + +1. Pydantic input and output validation. +2. Operation dispatch through `VidXPService.execute`. +3. Dependency selection without importing unrelated capabilities. +4. Index grouping and collection behavior when indexing is supported. +5. CLI behavior when specialized commands are provided. +6. Successful package metadata generation and a base-only import smoke test. + +Adding a normal capability must not require edits to +`src/vidxp/application.py`, `src/vidxp/core/runner.py`, or +`src/vidxp/core/storage.py`. diff --git a/docs/benchmarking/adapter_validation.md b/docs/benchmarking/adapter_validation.md index 3f0f80e..d314e2b 100644 --- a/docs/benchmarking/adapter_validation.md +++ b/docs/benchmarking/adapter_validation.md @@ -184,10 +184,10 @@ official-media run. ## Commands -Install the optional adapter parser: +Install the optional adapters with the capabilities they evaluate: ```powershell -python -m pip install -e ".[benchmarks]" +python -m pip install -e ".[scene,dialogue,benchmarks]" ``` Run a declared DiDeMo smoke subset by zero-based official annotation indices: diff --git a/docs/benchmarking/core_contract.md b/docs/benchmarking/core_contract.md index cde8151..e7581e8 100644 --- a/docs/benchmarking/core_contract.md +++ b/docs/benchmarking/core_contract.md @@ -33,9 +33,9 @@ does not change merely because the same run is relocated to another output or storage directory. Checkpoint filenames are hashes of video IDs, so official IDs cannot accidentally become platform-specific paths. -The existing CLI and Streamlit interface use `chroma_data/` as their single local -run for compatibility. Indexes created before schema version 2 must be rebuilt; -VidXP does not invent missing end timestamps, video IDs, or source IDs. +The CLI and Streamlit interface use `chroma_data/` as their local run. Indexes +created with an older schema must be rebuilt; VidXP does not invent missing end +timestamps, video IDs, or source IDs. ## Indexing API @@ -49,7 +49,7 @@ config = IndexConfig( run_id="clip-stride-5", enabled_modalities=("scene",), frame_stride=5, - scene_batch_size=32, + capability_options={"scene": {"batch_size": 32}}, storage_batch_size=256, ) @@ -126,7 +126,7 @@ an official ID cannot collide with the separator. ## Retrieval API ```python -from vidxp.core.search import search_scene +from vidxp.capabilities.scene.operations import search_scene result = search_scene( "a person cuts bread", diff --git a/docs/benchmarking/runtime_validation.md b/docs/benchmarking/runtime_validation.md index 04fbede..f18b4ae 100644 --- a/docs/benchmarking/runtime_validation.md +++ b/docs/benchmarking/runtime_validation.md @@ -34,7 +34,7 @@ python -m streamlit run .\src\vidxp\frontend.py For an installed wheel, the supported command remains: ```powershell -vidxp-ui +vidxp ui ``` ### Real execution checks @@ -49,7 +49,7 @@ selects the older installed TestPyPI package instead. | Check | Executed path | Observed result | |---|---|---| | Source Streamlit UI | Source selected through `PYTHONPATH`, real browser session | Page rendered with upload, index, status, and search controls; no import exception | -| Built-wheel UI | Fresh wheel installed without VidXP source on its import path; `vidxp-ui` opened in a real browser session | Page rendered successfully and the Streamlit health endpoint returned `ok` | +| Built-wheel UI | Fresh wheel installed without VidXP source on its import path; browser interface opened from the installed package | Page rendered successfully and the Streamlit health endpoint returned `ok` | | Built-wheel CLI | Final wheel installed into an isolated target while using the validated dependency environment; wheel target placed first on `PYTHONPATH` | Import resolved inside the wheel target and `python -m vidxp --help` listed the expected commands | | Dependency doctor | `vidxp doctor --modalities dialogue,scene,actor` | ChromaDB, MiniLM, CLIP, NumPy, OpenCV, Pillow, PyTorch, face recognition, MoviePy, WhisperX, and FFmpeg imports resolved | | Released-transcript path | Real MiniLM encoding, Chroma writes, and top-2 dialogue search over three timestamped segments | Run completed; two hits returned; first interval was `[0.0, 2.0]`; full run/source metadata present | diff --git a/docs/images/video-screenshot.jpeg b/docs/images/video-screenshot.jpeg new file mode 100644 index 0000000..53322b9 Binary files /dev/null and b/docs/images/video-screenshot.jpeg differ diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..ad876fe --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,32 @@ +# Release process + +VidXP publishes prerelease and stable packages from `main`. The old `release` +branch is retained only as historical ancestry for `v0.1.0`; it is not an active +publication branch. + +## Prereleases + +A release-relevant merge to `main` runs CI, lets Python Semantic Release +calculate the next version, creates a `b` prerelease tag and GitHub prerelease, +builds the distributions, and publishes them to TestPyPI. Commits that do not +require a semantic version bump do not publish a package. + +Pending Towncrier fragments remain in `changes/` during prereleases. + +## Stable releases + +1. Confirm that `main` is green and its TestPyPI prerelease is usable. +2. Confirm that every user-visible merged pull request has an accurate fragment. +3. Run the **Release (main → PyPI)** workflow from `main`. +4. Approve the `pypi` environment deployment when reviewer protection is enabled. +5. Confirm the new tag, GitHub release, PyPI package, and emptied pending + fragment set. + +The workflow only runs its validation and release jobs for a `main` dispatch. +Both jobs use the same immutable dispatch revision. Python Semantic Release is +the only version authority. During its build step, Towncrier receives that +calculated version, updates `CHANGELOG.md`, and removes the released fragments +before the release commit and tag are created. + +Release and CI tools are declared once in `utils/build-requirements.txt`. Do not +duplicate their versions in workflow files. diff --git a/pyproject.toml b/pyproject.toml index 496df25..02f6db6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["setuptools>=68", "wheel"] +requires = ["setuptools>=83", "wheel"] build-backend = "setuptools.build_meta" [project] name = "vidxp" -version = "0.1.0-b.3" +version = "0.1.0" authors = [ { name = "Muhammad Haroon" }, { name = "Talha Momin" }, @@ -33,34 +33,17 @@ classifiers = [ "Topic :: Multimedia :: Video", "Topic :: Software Development :: Libraries :: Python Modules", ] +dynamic = ["optional-dependencies"] dependencies = [ - "chromadb", - "face_recognition", "filelock>=3.13", - "moviepy==1.0.3", - "numpy>=2.1,<3", - "opencv-python", - "Pillow>=7.0.0", + "packaging>=24", + "pydantic>=2.8,<3", "rich", - "sentence-transformers>=3.4,<4", - "setuptools>=68,<81", - "torch", - "typer", - "whisperx>=3.8.6,<3.9", - "clip-anytorch==2.6.0", -] - -[project.optional-dependencies] -frontend = [ - "streamlit>=1.37", -] -benchmarks = [ - "srt>=3.5,<4", + "typer>=0.27,<1", ] [project.scripts] vidxp = "vidxp.cli:main" -vidxp-ui = "vidxp.frontend:main" [project.urls] Repository = "https://github.com/grayhatdevelopers/vidxp" @@ -75,12 +58,47 @@ where = ["src"] include = ["vidxp*"] [tool.setuptools.package-data] -vidxp = [] +vidxp = [ + "benchmarks/requirements.txt", + "capabilities/*/requirements.txt", + "requirements/*.txt", +] + +[tool.setuptools.dynamic.optional-dependencies] +storage = { file = ["src/vidxp/requirements/storage.txt"] } +dialogue = { file = [ + "src/vidxp/requirements/storage.txt", + "src/vidxp/capabilities/dialogue/requirements.txt", +] } +scene = { file = [ + "src/vidxp/requirements/storage.txt", + "src/vidxp/capabilities/scene/requirements.txt", +] } +actor = { file = [ + "src/vidxp/requirements/storage.txt", + "src/vidxp/capabilities/actor/requirements.txt", +] } +all = { file = [ + "src/vidxp/requirements/storage.txt", + "src/vidxp/capabilities/dialogue/requirements.txt", + "src/vidxp/capabilities/scene/requirements.txt", + "src/vidxp/capabilities/actor/requirements.txt", +] } +frontend = { file = ["src/vidxp/requirements/frontend.txt"] } +benchmarks = { file = ["src/vidxp/benchmarks/requirements.txt"] } + +[tool.ruff] +target-version = "py310" + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] [tool.semantic_release] version_toml = ["pyproject.toml:project.version"] tag_format = "v{version}" build_command = "bash ./utils/build_package.sh" +build_command_env = ["BUILD_CHANGELOG"] +assets = ["CHANGELOG.md"] commit_parser = "conventional" major_on_zero = true allow_zero_version = true @@ -92,15 +110,45 @@ patch_tags = ["fix", "perf"] parse_squash_commits = true ignore_merge_commits = true -[tool.semantic_release.changelog.default_templates] -changelog_file = "./CHANGELOG.md" -output_format = "md" - -[tool.semantic_release.branches.release] -match = "release" -prerelease = false - [tool.semantic_release.branches.main] match = "main" -prerelease = true +prerelease = false prerelease_token = "b" + +[tool.towncrier] +directory = "changes" +filename = "CHANGELOG.md" +start_string = "" +title_format = "## v{version} ({project_date})" +issue_format = "[#{issue}](https://github.com/grayhatdevelopers/vidxp/pull/{issue})" +wrap = false + +[[tool.towncrier.type]] +directory = "breaking" +name = "Breaking Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bug Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "deprecation" +name = "Deprecations" +showcontent = true + +[[tool.towncrier.type]] +directory = "docs" +name = "Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "security" +name = "Security" +showcontent = true diff --git a/src/vidxp/application.py b/src/vidxp/application.py new file mode 100644 index 0000000..57223d0 --- /dev/null +++ b/src/vidxp/application.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from typing import Any, Iterable, Mapping, cast + +from pydantic import BaseModel + +from vidxp.capabilities.contracts import ( + CapabilityContext, + PreparationContext, + capability_install_hint, +) +from vidxp.capabilities.registry import ( + capability_names, + collection_names, + dependency_checks, + get_capability, + index_capability_names, + preparable_capability_names, + validate_capability_options, + validate_capability_names, +) +from vidxp.capabilities.actor.schemas import ( + ActorClusterSummary, + ActorDetection, + ActorRenderResult, +) +from vidxp.capabilities.schemas import SearchResult +from vidxp.core.contracts import ( + CancellationToken, + IndexConfig, + IndexSchemaError, +) +from vidxp.core.manifest import ( + CHECKPOINT_DIRECTORY, + COMPLETION_FILE, + FAILURES_FILE, + MANIFEST_FILE, + TIMINGS_FILE, +) +from vidxp.core.runner import ( + ProgressCallback, + index_video, + indexing_in_progress, + local_config_from_status, +) +from vidxp.core.storage import IndexStorage +from vidxp.index_state import ( + INDEX_STATUS_FILE, + INDEX_STATUS_SCHEMA, + IndexingInProgressError, + read_index_status, + require_ready_index, +) + + +class VidXPService: + """Reusable application boundary for CLI, HTTP, and other adapters.""" + + def __init__( + self, + index_directory: str | Path = "chroma_data", + *, + device: str | None = None, + ) -> None: + self.index_directory = Path(index_directory) + self.device = device + + def index_status(self) -> dict[str, Any]: + status = read_index_status(self.index_directory) + if status is not None: + payload = dict(status) + else: + payload = { + "schema_version": INDEX_STATUS_SCHEMA, + "state": "missing", + "stage": "status", + "message": "No local video index was found.", + } + payload["index_directory"] = str(self.index_directory) + return payload + + def active_config(self) -> tuple[IndexConfig, dict[str, Any]]: + status = require_ready_index(self.index_directory) + config = local_config_from_status( + status, + storage_directory=self.index_directory, + ) + if self.device is not None: + config = replace(config, device=self.device) + return config, status + + def create_index( + self, + video_path: str | Path, + *, + modalities: Iterable[str] | None = None, + frame_stride: int = 1, + capability_options: Mapping[ + str, + Mapping[str, Any], + ] | None = None, + progress_callback: ProgressCallback | None = None, + cancellation: CancellationToken | None = None, + source_name: str | None = None, + ) -> dict[str, Any]: + selected = self._validate_modalities( + index_capability_names() if modalities is None else modalities + ) + non_indexable = [ + name + for name in selected + if get_capability(name).indexer is None + ] + if non_indexable: + raise ValueError( + "These capabilities do not support indexing: " + + ", ".join(non_indexable) + ) + options: dict[str, Any] = { + "enabled_modalities": selected, + "frame_stride": frame_stride, + "storage_directory": self.index_directory, + "collection_names": collection_names(selected), + "capability_options": validate_capability_options( + selected, + capability_options, + ), + } + if self.device is not None: + options["device"] = self.device + config = IndexConfig.local(**options) + return index_video( + str(video_path), + progress_callback=progress_callback, + source_name=source_name, + config=config, + cancellation=cancellation, + ) + + def indexing_in_progress(self) -> bool: + options: dict[str, Any] = { + "storage_directory": self.index_directory, + } + if self.device is not None: + options["device"] = self.device + return indexing_in_progress(IndexConfig.local(**options)) + + def check_dependencies( + self, + modalities: Iterable[str] | None = None, + ) -> dict[str, Any]: + selected = self._validate_modalities( + capability_names() if modalities is None else modalities + ) + checks = list(dependency_checks(selected)) + return { + "ok": all(check["ok"] for check in checks), + "modalities": list(selected), + "checks": checks, + } + + def prepare_models( + self, + modalities: Iterable[str] | None = None, + *, + capability_options: Mapping[ + str, + Mapping[str, Any], + ] | None = None, + progress_callback: ProgressCallback | None = None, + ) -> dict[str, Any]: + selected = self._validate_modalities( + preparable_capability_names() + if modalities is None + else modalities + ) + options = validate_capability_options(selected, capability_options) + checks = dependency_checks(selected) + failures = [check for check in checks if not check["ok"]] + if failures: + details = "; ".join( + f"{check['name']}: {check['error']}" + for check in failures + ) + extras = ",".join( + get_capability(name).extra for name in selected + ) + raise RuntimeError( + f"{details}. {capability_install_hint(extras)}" + ) + + prepared = [] + for name in selected: + capability = get_capability(name) + prepare = capability.prepare + if prepare is not None: + prepared.extend( + prepare( + PreparationContext( + device=self.device or "cpu", + settings=capability.config_model.model_validate( + options[name] + ), + ), + progress_callback, + ) + ) + return { + "prepared": prepared, + "modalities": list(selected), + "device": self.device or "cpu", + } + + def execute( + self, + capability: str, + operation: str, + payload: BaseModel | Mapping[str, Any], + ) -> BaseModel: + """Validate and execute a registered capability operation.""" + + definition = get_capability(capability) + try: + selected_operation = definition.operations[operation] + except KeyError as exc: + available = ", ".join(definition.operations) or "none" + raise ValueError( + f"Capability {capability!r} has no operation {operation!r}. " + f"Available operations: {available}." + ) from exc + config = None + if selected_operation.requires_index: + config, _ = self.active_config() + if capability not in config.enabled_modalities: + raise ValueError( + f"The {capability} capability is not present in this index." + ) + try: + return selected_operation.invoke( + CapabilityContext(config=config), + payload, + ) + except ModuleNotFoundError as exc: + dependency = exc.name or "optional dependency" + raise RuntimeError( + f"{dependency} is unavailable. " + + capability_install_hint(definition.extra) + ) from exc + + def search( + self, + modality: str, + query: str, + *, + top_k: int = 10, + ) -> SearchResult: + return cast( + SearchResult, + self.execute( + modality, + "search", + {"query": query, "top_k": top_k}, + ), + ) + + def actor_clusters(self) -> tuple[ActorClusterSummary, ...]: + result = self.execute("actor", "clusters", {}) + return tuple(result.clusters) + + def actor_detections( + self, + cluster_id: str, + ) -> list[ActorDetection]: + result = self.execute( + "actor", + "detections", + {"cluster_id": cluster_id}, + ) + return list(result.detections) + + def render_actor( + self, + cluster_id: str, + input_path: str | Path, + output_path: str | Path, + ) -> ActorRenderResult: + return cast( + ActorRenderResult, + self.execute( + "actor", + "render", + { + "cluster_id": cluster_id, + "input_path": input_path, + "output_path": output_path, + }, + ), + ) + + def clear_index(self) -> bool: + if not self.index_directory.exists(): + return False + base_config = IndexConfig.local( + storage_directory=self.index_directory, + enabled_modalities=index_capability_names(), + collection_names=collection_names(), + ) + if indexing_in_progress(base_config): + raise IndexingInProgressError( + f"Indexing is active for {self.index_directory}." + ) + + status = read_index_status(self.index_directory) + if status is not None and status.get("state") == "ready": + try: + config = local_config_from_status( + status, + storage_directory=self.index_directory, + ) + except (IndexSchemaError, KeyError, TypeError, ValueError): + config = base_config + else: + config = base_config + try: + with IndexStorage(config) as storage: + storage.clear() + except ModuleNotFoundError as exc: + dependency = exc.name or "optional storage dependency" + raise RuntimeError( + f"{dependency} is unavailable. " + + capability_install_hint("storage") + ) from exc + + for name in ( + INDEX_STATUS_FILE, + MANIFEST_FILE, + TIMINGS_FILE, + FAILURES_FILE, + COMPLETION_FILE, + ): + (self.index_directory / name).unlink(missing_ok=True) + checkpoint_directory = self.index_directory / CHECKPOINT_DIRECTORY + if checkpoint_directory.is_dir(): + for checkpoint in checkpoint_directory.glob("*.json"): + checkpoint.unlink() + try: + checkpoint_directory.rmdir() + except OSError: + pass + return True + + @staticmethod + def _validate_modalities( + modalities: Iterable[str], + ) -> tuple[str, ...]: + return validate_capability_names(modalities) diff --git a/src/vidxp/benchmarks/cli.py b/src/vidxp/benchmarks/cli.py index eb77b82..fd84ca0 100644 --- a/src/vidxp/benchmarks/cli.py +++ b/src/vidxp/benchmarks/cli.py @@ -5,13 +5,19 @@ from typing import Annotated, Literal import typer -from rich import print +from rich import print as rich_print from vidxp.benchmarks.didemo import run_didemo from vidxp.benchmarks.hirest import ( HIREST_DEFAULT_WINDOW_FRACTION, run_hirest, ) +from vidxp.cli_support import ( + OutputFormat, + effective_output_format, + emit_json, + state_from_context, +) app = typer.Typer(help="Run official benchmark adapters.") @@ -62,6 +68,7 @@ def _pair_file(path: Path | None) -> list[tuple[str, str]] | None: @app.command("didemo") def didemo_command( + ctx: typer.Context, annotations: Annotated[Path, typer.Option(exists=True, dir_okay=False)], evaluator: Annotated[Path, typer.Option(exists=True, dir_okay=False)], media_directory: Annotated[ @@ -91,9 +98,14 @@ def didemo_command( ] = "max", frame_stride: Annotated[int, typer.Option(min=1)] = 1, reset: Annotated[bool, typer.Option()] = False, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, ) -> None: """Run DiDeMo scene retrieval and its official evaluator.""" + state = state_from_context(ctx) metrics = run_didemo( annotations_path=annotations, evaluator_path=evaluator, @@ -105,12 +117,17 @@ def didemo_command( split=split, chunk_pooling=chunk_pooling, reset=reset, + device=state.service.device or "cpu", ) - print(metrics) + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(metrics) + else: + rich_print(metrics) @app.command("hirest") def hirest_command( + ctx: typer.Context, ground_truth: Annotated[Path, typer.Option(exists=True, dir_okay=False)], categories: Annotated[Path, typer.Option(exists=True, dir_okay=False)], evaluator: Annotated[Path, typer.Option(exists=True, dir_okay=False)], @@ -145,9 +162,14 @@ def hirest_command( ), ] = HIREST_DEFAULT_WINDOW_FRACTION, reset: Annotated[bool, typer.Option()] = False, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, ) -> None: """Run HiREST released-ASR retrieval; score validation predictions.""" + state = state_from_context(ctx) metrics = run_hirest( ground_truth_path=ground_truth, categories_path=categories, @@ -160,5 +182,9 @@ def hirest_command( split=split, temporal_window_fraction=temporal_window_fraction, reset=reset, + device=state.service.device or "cpu", ) - print(metrics) + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(metrics) + else: + rich_print(metrics) diff --git a/src/vidxp/benchmarks/didemo.py b/src/vidxp/benchmarks/didemo.py index 3917555..bdf6752 100644 --- a/src/vidxp/benchmarks/didemo.py +++ b/src/vidxp/benchmarks/didemo.py @@ -16,10 +16,11 @@ run_logged_evaluator, verify_artifact, ) -from vidxp.core.contracts import IndexConfig, SearchHit, VideoSource +from vidxp.capabilities.scene.operations import search_scene +from vidxp.capabilities.schemas import SearchHit +from vidxp.core.contracts import IndexConfig, VideoSource from vidxp.core.manifest import write_json_atomic from vidxp.core.runner import run_index -from vidxp.core.search import search_scene DIDEMO_REVISION = "b6a555c8134581305d0ed4716fbc192860e0b88c" @@ -354,6 +355,7 @@ def run_didemo( annotation_indices: Sequence[int] | None = None, media_overrides: Mapping[str, str | Path] | None = None, frame_stride: int = 1, + device: str = "cpu", split: Literal["validation", "test"] = "test", chunk_pooling: Literal["max", "mean"] = "max", reset: bool = False, @@ -377,6 +379,7 @@ def run_didemo( run_id=run_id, enabled_modalities=("scene",), frame_stride=frame_stride, + device=device, output_root=output_root, ) run_directory = config.run_directory diff --git a/src/vidxp/benchmarks/hirest.py b/src/vidxp/benchmarks/hirest.py index 3c09d56..ce499a3 100644 --- a/src/vidxp/benchmarks/hirest.py +++ b/src/vidxp/benchmarks/hirest.py @@ -18,10 +18,12 @@ run_logged_evaluator, verify_artifact, ) -from vidxp.core.contracts import IndexConfig, SearchHit, VideoSource +from vidxp.capabilities.dialogue.config import dialogue_config +from vidxp.capabilities.dialogue.operations import search_dialogue +from vidxp.capabilities.schemas import SearchHit +from vidxp.core.contracts import IndexConfig, VideoSource from vidxp.core.manifest import sha256_file, write_json_atomic from vidxp.core.runner import run_index -from vidxp.core.search import search_dialogue HIREST_REVISION = "deffc169b4e8d51c1589d5512ad05da61e81bcee" @@ -401,6 +403,7 @@ def run_hirest( pairs: Sequence[tuple[str, str]] | None = None, split: Literal["validation", "test"] = "test", temporal_window_fraction: float = HIREST_DEFAULT_WINDOW_FRACTION, + device: str = "cpu", reset: bool = False, ) -> dict[str, Any]: if split not in {"validation", "test"}: @@ -425,6 +428,7 @@ def run_hirest( split=split, run_id=run_id, enabled_modalities=("dialogue",), + device=device, output_root=output_root, ) run_directory = config.run_directory @@ -476,7 +480,7 @@ def run_hirest( "prediction_format_validated": True, "input_mode": "released_timestamped_asr", "dialogue_words_per_phrase": ( - config.dialogue_words_per_phrase + dialogue_config(config).words_per_phrase ), "segment_word_timestamps": ( "linear_interpolation_within_srt_cue" diff --git a/src/vidxp/benchmarks/requirements.txt b/src/vidxp/benchmarks/requirements.txt new file mode 100644 index 0000000..7b6efb9 --- /dev/null +++ b/src/vidxp/benchmarks/requirements.txt @@ -0,0 +1 @@ +srt>=3.5,<4 diff --git a/src/vidxp/capabilities/__init__.py b/src/vidxp/capabilities/__init__.py new file mode 100644 index 0000000..381c275 --- /dev/null +++ b/src/vidxp/capabilities/__init__.py @@ -0,0 +1,15 @@ +"""Built-in VidXP capabilities and their explicit registry.""" + +from vidxp.capabilities.registry import ( + CAPABILITIES, + capability_names, + get_capability, + index_capability_names, +) + +__all__ = [ + "CAPABILITIES", + "capability_names", + "get_capability", + "index_capability_names", +] diff --git a/src/vidxp/capabilities/actor/__init__.py b/src/vidxp/capabilities/actor/__init__.py new file mode 100644 index 0000000..e58f1ec --- /dev/null +++ b/src/vidxp/capabilities/actor/__init__.py @@ -0,0 +1 @@ +"""Actor capability implementation.""" diff --git a/src/vidxp/capabilities/actor/cli.py b/src/vidxp/capabilities/actor/cli.py new file mode 100644 index 0000000..51767ed --- /dev/null +++ b/src/vidxp/capabilities/actor/cli.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Iterable + +import typer +from rich.console import Console +from rich.table import Table + +from vidxp.cli_support import ( + CLIState, + OutputFormat, + effective_output_format, + emit_json, + state_from_context, +) + + +app = typer.Typer( + no_args_is_help=True, + help="Inspect and render actor clusters.", +) + + +def complete_cluster( + ctx: typer.Context, + incomplete: str, +) -> Iterable[tuple[str, str]]: + state = ctx.find_root().obj + if not isinstance(state, CLIState): + return + try: + clusters = state.service.actor_clusters() + except Exception: + return + for cluster in clusters: + if cluster.cluster_id.startswith(incomplete): + yield ( + cluster.cluster_id, + f"{cluster.detection_count} detections", + ) + + +@app.command("list") +def actors_list( + ctx: typer.Context, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """List actor clusters in the selected index.""" + + state = state_from_context(ctx) + clusters = state.service.actor_clusters() + payload = { + "clusters": [cluster.to_dict() for cluster in clusters], + "count": len(clusters), + } + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(payload) + return + table = Table(title="Actor clusters") + table.add_column("Cluster") + table.add_column("Detections", justify="right") + table.add_column("First", justify="right") + table.add_column("Last", justify="right") + for cluster in clusters: + table.add_row( + cluster.cluster_id, + str(cluster.detection_count), + f"{cluster.first_timestamp:.3f}s", + f"{cluster.last_timestamp:.3f}s", + ) + Console().print(table) + + +@app.command("inspect") +def actors_inspect( + ctx: typer.Context, + cluster_id: Annotated[ + str, + typer.Argument( + autocompletion=complete_cluster, + help="Actor cluster identifier.", + ), + ], + limit: Annotated[ + int, + typer.Option("--limit", min=1, help="Maximum detections to display."), + ] = 20, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Inspect retained detections for one actor cluster.""" + + state = state_from_context(ctx) + detections = state.service.actor_detections(cluster_id) + if not detections: + raise typer.BadParameter(f"Actor cluster {cluster_id} was not found.") + payload = { + "cluster_id": cluster_id, + "detection_count": len(detections), + "detections": [ + detection.model_dump(mode="json") + for detection in detections[:limit] + ], + "truncated": len(detections) > limit, + } + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(payload) + return + table = Table(title=f"Actor cluster {cluster_id}") + table.add_column("Frame", justify="right") + table.add_column("Timestamp", justify="right") + table.add_column("Detection") + for detection in detections[:limit]: + table.add_row( + str(detection.frame_index), + f"{detection.timestamp:.3f}s", + detection.detection_id, + ) + Console().print(table) + if len(detections) > limit: + typer.echo(f"Showing {limit} of {len(detections)} detections.") + + +@app.command("render") +def actors_render( + ctx: typer.Context, + cluster_id: Annotated[ + str, + typer.Argument( + autocompletion=complete_cluster, + help="Actor cluster identifier.", + ), + ], + input_path: Annotated[ + Path, + typer.Argument( + exists=True, + dir_okay=False, + readable=True, + resolve_path=True, + help="Source video used to create the active index.", + ), + ], + output_path: Annotated[ + Path, + typer.Option( + "--output", + "-o", + dir_okay=False, + help="Rendered video destination.", + ), + ] = Path("output.mp4"), + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Render one actor cluster as a result video.""" + + state = state_from_context(ctx) + result = state.service.render_actor( + cluster_id, + input_path, + output_path, + ) + payload = { + "cluster_id": cluster_id, + "output_path": str(result.output_path), + "detection_count": result.detection_count, + } + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(payload) + else: + typer.secho( + f"Video saved as {result.output_path}", + fg=typer.colors.GREEN, + ) diff --git a/src/vidxp/capabilities/actor/config.py b/src/vidxp/capabilities/actor/config.py new file mode 100644 index 0000000..8756fd0 --- /dev/null +++ b/src/vidxp/capabilities/actor/config.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from pydantic import Field + +from vidxp.capabilities.contracts import CapabilityConfig +from vidxp.core.contracts import IndexConfig + + +class ActorConfig(CapabilityConfig): + batch_size: int = Field(default=16, gt=0) + match_threshold: float = Field(default=0.55, gt=0, lt=1) + num_jitters: int = Field(default=2, gt=0) + minimum_detections: int = Field(default=4, gt=0) + + +def actor_config(config: IndexConfig) -> ActorConfig: + return ActorConfig.model_validate(config.options_for("actor")) diff --git a/src/vidxp/capabilities/actor/definition.py b/src/vidxp/capabilities/actor/definition.py new file mode 100644 index 0000000..690005d --- /dev/null +++ b/src/vidxp/capabilities/actor/definition.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from vidxp.capabilities.actor.config import ActorConfig, actor_config +from vidxp.capabilities.actor.indexing import VISUAL_PROCESSOR +from vidxp.capabilities.actor.operations import ( + clusters_operation, + detections_operation, + render_operation, +) +from vidxp.capabilities.contracts import ( + CapabilityDefinition, + OperationDefinition, +) +from vidxp.capabilities.actor.schemas import ( + ActorClustersInput, + ActorClustersOutput, + ActorDetectionsInput, + ActorDetectionsOutput, + ActorRenderInput, + ActorRenderResult, +) +from vidxp.capabilities.visual import index_capabilities +from vidxp.core.contracts import IndexConfig, VideoSource + +def model_manifest( + config: IndexConfig, + _sources: tuple[VideoSource, ...], +) -> Mapping[str, Any]: + settings = actor_config(config) + return { + "actor": { + "library": "face_recognition", + "match_threshold": settings.match_threshold, + "num_jitters": settings.num_jitters, + "minimum_detections": settings.minimum_detections, + } + } + + +def cli_app(): + from vidxp.capabilities.actor.cli import app + + return app + + +DEFINITION = CapabilityDefinition( + name="actor", + description="Index, inspect, and render actor clusters.", + extra="actor", + config_model=ActorConfig, + collection_name="actor", + indexer=index_capabilities, + index_processor=VISUAL_PROCESSOR, + index_stage="visual_indexing", + model_manifest=model_manifest, + operations={ + "clusters": OperationDefinition( + input_model=ActorClustersInput, + output_model=ActorClustersOutput, + handler=clusters_operation, + ), + "detections": OperationDefinition( + input_model=ActorDetectionsInput, + output_model=ActorDetectionsOutput, + handler=detections_operation, + ), + "render": OperationDefinition( + input_model=ActorRenderInput, + output_model=ActorRenderResult, + handler=render_operation, + ), + }, + cli_name="actors", + cli_factory=cli_app, +) diff --git a/src/vidxp/core/indexing_actor.py b/src/vidxp/capabilities/actor/indexing.py similarity index 71% rename from src/vidxp/core/indexing_actor.py rename to src/vidxp/capabilities/actor/indexing.py index 7f4a9ce..7328d10 100644 --- a/src/vidxp/core/indexing_actor.py +++ b/src/vidxp/capabilities/actor/indexing.py @@ -3,6 +3,7 @@ from dataclasses import dataclass, field from typing import Any +from vidxp.capabilities.actor.config import actor_config from vidxp.core.contracts import ( CancellationToken, IndexConfig, @@ -10,6 +11,7 @@ batched, stable_source_id, ) +from vidxp.core.indexing_common import ProgressCallback from vidxp.core.storage import IndexStorage @@ -79,7 +81,8 @@ def process_actor_samples( import face_recognition import numpy as np - for group in batched(samples, config.actor_batch_size): + settings = actor_config(config) + for group in batched(samples, settings.batch_size): cancellation.raise_if_cancelled() detections = [] for sample in group: @@ -88,7 +91,7 @@ def process_actor_samples( encodings = face_recognition.face_encodings( sample.frame, locations, - num_jitters=config.face_num_jitters, + num_jitters=settings.num_jitters, ) for ordinal, (encoding, location) in enumerate( zip(encodings, locations) @@ -97,7 +100,7 @@ def process_actor_samples( face_recognition, state.known_encodings, encoding, - config.face_match_threshold, + settings.match_threshold, ) if match is None: cluster_id = str(len(state.known_ids) + 1) @@ -140,16 +143,75 @@ def finalize_actor_index( config: IndexConfig, storage: IndexStorage, ) -> tuple[int, int]: + settings = actor_config(config) rejected = [ cluster_id for cluster_id, size in state.cluster_sizes.items() - if size < config.actor_min_detections + if size < settings.minimum_detections ] for cluster_id in rejected: - storage.delete_actor_cluster(str(config.video_id), cluster_id) + storage.delete_records( + "actor", + video_id=str(config.video_id), + filters={"cluster_id": cluster_id}, + ) retained = { cluster_id: size for cluster_id, size in state.cluster_sizes.items() - if size >= config.actor_min_detections + if size >= settings.minimum_detections } return sum(retained.values()), len(retained) + + +class ActorVisualProcessor: + def batch_size(self, config: IndexConfig) -> int: + return actor_config(config).batch_size + + def prepare( + self, + config: IndexConfig, + progress: ProgressCallback | None, + ) -> ActorIndexState: + return ActorIndexState() + + def process( + self, + samples, + *, + state: ActorIndexState, + info, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, + ) -> None: + process_actor_samples( + samples, + state=state, + config=config, + storage=storage, + cancellation=cancellation, + ) + + def finalize( + self, + state: ActorIndexState, + *, + config: IndexConfig, + storage: IndexStorage, + ) -> tuple[dict[str, Any], int]: + detections, clusters = finalize_actor_index( + state, + config=config, + storage=storage, + ) + return ( + { + "actor_frames": state.processed_frames, + "actor_detections": detections, + "actor_clusters": clusters, + }, + state.processed_frames, + ) + + +VISUAL_PROCESSOR = ActorVisualProcessor() diff --git a/src/vidxp/capabilities/actor/operations.py b/src/vidxp/capabilities/actor/operations.py new file mode 100644 index 0000000..88b7276 --- /dev/null +++ b/src/vidxp/capabilities/actor/operations.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from vidxp.capabilities.actor.results import ( + actor_clusters, + actor_detections, + render_actor_result, +) +from vidxp.capabilities.contracts import CapabilityContext +from vidxp.capabilities.actor.schemas import ( + ActorClustersInput, + ActorClustersOutput, + ActorDetectionsInput, + ActorDetectionsOutput, + ActorRenderInput, + ActorRenderResult, +) + + +def clusters_operation( + context: CapabilityContext, + _request: ActorClustersInput, +) -> ActorClustersOutput: + return ActorClustersOutput( + clusters=actor_clusters(context.require_config()) + ) + + +def detections_operation( + context: CapabilityContext, + request: ActorDetectionsInput, +) -> ActorDetectionsOutput: + config = context.require_config() + return ActorDetectionsOutput( + cluster_id=request.cluster_id, + detections=actor_detections( + config, + request.cluster_id, + ), + ) + + +def render_operation( + context: CapabilityContext, + request: ActorRenderInput, +) -> ActorRenderResult: + config = context.require_config() + return render_actor_result( + config, + request.cluster_id, + request.input_path, + request.output_path, + ) diff --git a/src/vidxp/capabilities/actor/requirements.txt b/src/vidxp/capabilities/actor/requirements.txt new file mode 100644 index 0000000..383f8ac --- /dev/null +++ b/src/vidxp/capabilities/actor/requirements.txt @@ -0,0 +1,3 @@ +face-recognition +numpy>=2.1,<3 +opencv-python diff --git a/src/vidxp/capabilities/actor/results.py b/src/vidxp/capabilities/actor/results.py new file mode 100644 index 0000000..1ab4386 --- /dev/null +++ b/src/vidxp/capabilities/actor/results.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from pathlib import Path + +from vidxp.capabilities.actor.schemas import ( + ActorClusterSummary, + ActorDetection, + ActorRenderResult, +) +from vidxp.core.contracts import IndexConfig +from vidxp.core.storage import IndexStorage +from vidxp.core.video import render_actor_video + + +class ActorClusterNotFoundError(LookupError): + """Raised when an actor cluster has no retained detections.""" + + +def actor_clusters( + config: IndexConfig, + *, + storage: IndexStorage | None = None, +) -> tuple[ActorClusterSummary, ...]: + if config.video_id is None: + raise ValueError("IndexConfig.video_id is required for actor results.") + owns_storage = storage is None + active_storage = storage or IndexStorage(config) + try: + records = active_storage.records( + "actor", + video_id=config.video_id, + ) + finally: + if owns_storage: + active_storage.close() + + grouped: dict[str, list[dict]] = {} + for record in records: + grouped.setdefault(str(record["cluster_id"]), []).append(record) + return tuple( + ActorClusterSummary( + cluster_id=cluster_id, + video_id=config.video_id, + detection_count=len(cluster_records), + first_timestamp=min( + float(record["timestamp"]) for record in cluster_records + ), + last_timestamp=max( + float(record["timestamp"]) for record in cluster_records + ), + ) + for cluster_id, cluster_records in sorted(grouped.items()) + ) + + +def actor_detections( + config: IndexConfig, + cluster_id: str, + *, + storage: IndexStorage | None = None, +) -> list[ActorDetection]: + if config.video_id is None: + raise ValueError("IndexConfig.video_id is required for actor results.") + owns_storage = storage is None + active_storage = storage or IndexStorage(config) + try: + records = active_storage.records( + "actor", + video_id=config.video_id, + filters={"cluster_id": cluster_id}, + ) + finally: + if owns_storage: + active_storage.close() + + detections = [ + ActorDetection( + **{ + key: value + for key, value in record.items() + if not key.startswith("bbox_") + }, + bbox=( + int(record["bbox_top"]), + int(record["bbox_right"]), + int(record["bbox_bottom"]), + int(record["bbox_left"]), + ), + ) + for record in records + ] + return sorted( + detections, + key=lambda item: (item.frame_index, item.detection_id), + ) + + +def render_actor_result( + config: IndexConfig, + cluster_id: str, + input_path: str | Path, + output_path: str | Path, + *, + storage: IndexStorage | None = None, +) -> ActorRenderResult: + detections = actor_detections(config, cluster_id, storage=storage) + if not detections: + raise ActorClusterNotFoundError( + f"Actor cluster {cluster_id} was not found in the completed index." + ) + destination = Path(output_path) + render_actor_video(input_path, destination, cluster_id, detections) + return ActorRenderResult( + output_path=destination, + detection_count=len(detections), + ) diff --git a/src/vidxp/capabilities/actor/schemas.py b/src/vidxp/capabilities/actor/schemas.py new file mode 100644 index 0000000..26fcea8 --- /dev/null +++ b/src/vidxp/capabilities/actor/schemas.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from pathlib import Path + +from pydantic import Field + +from vidxp.capabilities.contracts import CapabilityInput, CapabilityOutput + + +class ActorClustersInput(CapabilityInput): + pass + + +class ActorClusterSummary(CapabilityOutput): + cluster_id: str = Field(min_length=1) + video_id: str = Field(min_length=1) + detection_count: int = Field(ge=0) + first_timestamp: float = Field(ge=0) + last_timestamp: float = Field(ge=0) + + def to_dict(self) -> dict: + return self.model_dump(mode="json") + + +class ActorClustersOutput(CapabilityOutput): + clusters: tuple[ActorClusterSummary, ...] = () + + +class ActorDetectionsInput(CapabilityInput): + cluster_id: str = Field(min_length=1) + + +class ActorDetection(CapabilityOutput): + detection_id: str = Field(min_length=1) + cluster_id: str = Field(min_length=1) + frame_index: int = Field(ge=0) + timestamp: float = Field(ge=0) + bbox: tuple[int, int, int, int] + dataset: str + split: str + run_id: str + video_id: str + modality: str + source_id: str + + +class ActorDetectionsOutput(CapabilityOutput): + cluster_id: str = Field(min_length=1) + detections: tuple[ActorDetection, ...] = () + + +class ActorRenderInput(CapabilityInput): + cluster_id: str = Field(min_length=1) + input_path: Path + output_path: Path + + +class ActorRenderResult(CapabilityOutput): + output_path: Path + detection_count: int = Field(gt=0) diff --git a/src/vidxp/capabilities/contracts.py b/src/vidxp/capabilities/contracts.py new file mode 100644 index 0000000..d022cde --- /dev/null +++ b/src/vidxp/capabilities/contracts.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +from types import MappingProxyType +from typing import Any, Callable, Mapping + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + NonNegativeFloat, + field_validator, + model_validator, +) +from packaging.requirements import Requirement + +from vidxp.core.contracts import IndexConfig, VideoSource +from vidxp.core.indexing_common import ProgressCallback + + +class _ContractModel(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra="forbid", + frozen=True, + ) + + +class CapabilityInput(BaseModel): + """Base model for validated capability input.""" + + model_config = ConfigDict(extra="forbid") + + +class CapabilityOutput(_ContractModel): + """Base model for validated capability output.""" + + +class CapabilityConfig(_ContractModel): + """Base model for settings owned and validated by one capability.""" + + +class RuntimeCheck(_ContractModel): + """One non-package environment requirement.""" + + label: str = Field(min_length=1) + check: Callable[[], str | None] + applies_to: Callable[[VideoSource], bool] | None = None + + def inspect(self) -> dict[str, Any]: + try: + detail = self.check() + except Exception as exc: + return { + "name": self.label, + "ok": False, + "error": f"{type(exc).__name__}: {exc}", + } + result: dict[str, Any] = { + "name": self.label, + "ok": True, + "error": None, + } + if detail is not None: + result["path"] = detail + return result + + def applies(self, source: VideoSource | None) -> bool: + return ( + source is None + or self.applies_to is None + or self.applies_to(source) + ) + + +class CapabilityContext(_ContractModel): + """Runtime context shared by transport-neutral capability operations.""" + + config: IndexConfig | None + + def require_config(self) -> IndexConfig: + if self.config is None: + raise RuntimeError("This operation requires an active index.") + return self.config + + +class PreparationContext(_ContractModel): + """Runtime values supplied to one capability's preparation hook.""" + + device: str = Field(min_length=1) + settings: CapabilityConfig + + +OperationHandler = Callable[[CapabilityContext, BaseModel], BaseModel | Mapping] + + +class OperationDefinition(_ContractModel): + """Validated input, output, and implementation for one operation.""" + + input_model: type[BaseModel] + output_model: type[BaseModel] + handler: OperationHandler + requires_index: bool = True + + @field_validator("input_model", "output_model") + @classmethod + def _require_model( + cls, + value: type[BaseModel], + ) -> type[BaseModel]: + if not isinstance(value, type) or not issubclass(value, BaseModel): + raise ValueError("Operation schemas must be Pydantic models.") + return value + + def invoke( + self, + context: CapabilityContext, + payload: BaseModel | Mapping[str, Any], + ) -> BaseModel: + request = self.input_model.model_validate(payload) + result = self.handler(context, request) + return self.output_model.model_validate(result) + + +class CapabilityIndexResult(_ContractModel): + """Summary and timing data returned by an indexing handler.""" + + summary: dict[str, Any] + timings: dict[str, NonNegativeFloat] = Field(default_factory=dict) + + +IndexHandler = Callable[..., CapabilityIndexResult] +PrepareHandler = Callable[ + [PreparationContext, ProgressCallback | None], + tuple[str, ...], +] +RequirementFilter = Callable[ + [VideoSource, tuple[Requirement, ...]], + tuple[Requirement, ...], +] +ModelManifest = Callable[ + [IndexConfig, tuple[VideoSource, ...]], + Mapping[str, Any], +] +CLIFactory = Callable[[], Any] + + +class CapabilityDefinition(_ContractModel): + """Everything the application needs to run one named capability.""" + + name: str = Field(min_length=1) + description: str = Field(min_length=1) + extra: str = Field(min_length=1) + config_model: type[CapabilityConfig] = CapabilityConfig + runtime_checks: tuple[RuntimeCheck, ...] = () + collection_name: str | None = None + indexer: IndexHandler | None = None + index_processor: Any | None = None + index_stage: str | None = None + operations: Mapping[str, OperationDefinition] = Field(default_factory=dict) + requirement_filter: RequirementFilter | None = None + prepare: PrepareHandler | None = None + model_manifest: ModelManifest | None = None + cli_name: str | None = None + cli_factory: CLIFactory | None = None + + @field_validator("config_model") + @classmethod + def _require_config_model( + cls, + value: type[CapabilityConfig], + ) -> type[CapabilityConfig]: + if ( + not isinstance(value, type) + or not issubclass(value, CapabilityConfig) + ): + raise ValueError( + "Capability config_model must extend CapabilityConfig." + ) + return value + + @field_validator("operations") + @classmethod + def _freeze_operations( + cls, + value: Mapping[str, OperationDefinition], + ) -> Mapping[str, OperationDefinition]: + return MappingProxyType(dict(value)) + + @model_validator(mode="after") + def _require_complete_integrations(self) -> CapabilityDefinition: + indexing_fields = ( + self.collection_name, + self.indexer, + self.index_stage, + ) + if any(value is not None for value in indexing_fields) and not all( + value is not None for value in indexing_fields + ): + raise ValueError( + "Indexable capabilities must declare collection names, " + "an indexer, and an index stage together." + ) + if (self.cli_name is None) != (self.cli_factory is None): + raise ValueError( + "cli_name and cli_factory must either both be set or both be unset." + ) + if self.indexer is None and not self.operations: + raise ValueError( + "A capability must provide an indexer or at least one operation." + ) + return self + + def source_requirements( + self, + source: VideoSource, + requirements: tuple[Requirement, ...], + ) -> tuple[Requirement, ...]: + if self.requirement_filter is None: + return requirements + return self.requirement_filter(source, requirements) + + +def capability_install_hint(name: str) -> str: + return f'Install the capability with: pip install "vidxp[{name}]"' diff --git a/src/vidxp/capabilities/dialogue/__init__.py b/src/vidxp/capabilities/dialogue/__init__.py new file mode 100644 index 0000000..0573c3c --- /dev/null +++ b/src/vidxp/capabilities/dialogue/__init__.py @@ -0,0 +1 @@ +"""Dialogue capability implementation.""" diff --git a/src/vidxp/capabilities/dialogue/config.py b/src/vidxp/capabilities/dialogue/config.py new file mode 100644 index 0000000..3999801 --- /dev/null +++ b/src/vidxp/capabilities/dialogue/config.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from pydantic import Field + +from vidxp.capabilities.contracts import CapabilityConfig +from vidxp.core.contracts import IndexConfig + + +class DialogueConfig(CapabilityConfig): + words_per_phrase: int = Field(default=5, gt=0) + embedding_batch_size: int = Field(default=128, gt=0) + transcription_batch_size: int = Field(default=16, gt=0) + normalize_embeddings: bool = True + sentence_model: str = Field( + default="sentence-transformers/all-MiniLM-L6-v2", + min_length=1, + ) + whisper_model: str = Field(default="large-v2", min_length=1) + alignment_language: str | None = Field(default=None, min_length=1) + + +def dialogue_config(config: IndexConfig) -> DialogueConfig: + return DialogueConfig.model_validate(config.options_for("dialogue")) diff --git a/src/vidxp/capabilities/dialogue/definition.py b/src/vidxp/capabilities/dialogue/definition.py new file mode 100644 index 0000000..f7682e5 --- /dev/null +++ b/src/vidxp/capabilities/dialogue/definition.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +from vidxp.capabilities.contracts import ( + CapabilityDefinition, + OperationDefinition, + PreparationContext, + RuntimeCheck, +) +from vidxp.capabilities.dialogue.config import DialogueConfig, dialogue_config +from vidxp.capabilities.dialogue.models import ( + get_alignment_model, + get_embedder, + get_whisper_model, +) +from vidxp.capabilities.dialogue.operations import ( + index_capability, + search_operation, +) +from vidxp.capabilities.schemas import SearchInput, SearchResult +from vidxp.core.contracts import IndexConfig, VideoSource +from vidxp.core.indexing_common import ProgressCallback +from vidxp.core.video import ffmpeg_binary + + +FFMPEG = RuntimeCheck( + label="FFmpeg", + check=ffmpeg_binary, + applies_to=lambda source: source.transcript is None, +) + + +def filter_requirements_for_source( + source: VideoSource, + requirements: tuple[Requirement, ...], +) -> tuple[Requirement, ...]: + if source.transcript is not None: + needed = {"chromadb", "sentence-transformers"} + return tuple( + requirement + for requirement in requirements + if canonicalize_name(requirement.name) in needed + ) + return requirements + + +def prepare_models( + context: PreparationContext, + progress: ProgressCallback | None, +) -> tuple[str, ...]: + settings = DialogueConfig.model_validate(context.settings) + prepared = [] + + def report(stage: str, message: str) -> None: + if progress is not None: + progress( + { + "state": "preparing", + "stage": stage, + "message": message, + } + ) + + report( + "dialogue_model", + f"Preparing dialogue model: {settings.sentence_model}", + ) + get_embedder(settings.sentence_model, context.device) + prepared.append(settings.sentence_model) + report( + "transcription_model", + f"Preparing transcription model: WhisperX {settings.whisper_model}", + ) + get_whisper_model(settings.whisper_model, context.device) + prepared.append(settings.whisper_model) + if settings.alignment_language: + report( + "alignment_model", + f"Preparing the {settings.alignment_language} alignment model.", + ) + get_alignment_model(settings.alignment_language, context.device) + prepared.append( + f"whisperx-alignment:{settings.alignment_language}" + ) + return tuple(prepared) + + +def model_manifest( + config: IndexConfig, + sources: tuple[VideoSource, ...], +) -> Mapping[str, Any]: + settings = dialogue_config(config) + result: dict[str, Any] = {"dialogue": settings.sentence_model} + if any(source.transcript is None for source in sources): + result["transcription"] = settings.whisper_model + return result + + +DEFINITION = CapabilityDefinition( + name="dialogue", + description="Index and search spoken dialogue.", + extra="dialogue", + config_model=DialogueConfig, + collection_name="dialogue", + indexer=index_capability, + index_stage="dialogue_indexing", + runtime_checks=(FFMPEG,), + requirement_filter=filter_requirements_for_source, + prepare=prepare_models, + model_manifest=model_manifest, + operations={ + "search": OperationDefinition( + input_model=SearchInput, + output_model=SearchResult, + handler=search_operation, + ) + }, +) diff --git a/src/vidxp/core/indexing_dialogue.py b/src/vidxp/capabilities/dialogue/indexing.py similarity index 90% rename from src/vidxp/core/indexing_dialogue.py rename to src/vidxp/capabilities/dialogue/indexing.py index e1c44c4..4e5201c 100644 --- a/src/vidxp/core/indexing_dialogue.py +++ b/src/vidxp/capabilities/dialogue/indexing.py @@ -5,6 +5,12 @@ from pathlib import Path from typing import Any, Mapping, Sequence +from vidxp.capabilities.dialogue.config import dialogue_config +from vidxp.capabilities.dialogue.models import ( + get_alignment_model, + get_embedder, + get_whisper_model, +) from vidxp.core.contracts import ( CancellationToken, IndexConfig, @@ -13,11 +19,6 @@ stable_source_id, ) from vidxp.core.indexing_common import ProgressCallback, report_progress -from vidxp.core.models import ( - get_alignment_model, - get_embedder, - get_whisper_model, -) from vidxp.core.storage import IndexStorage from vidxp.core.video import extract_audio @@ -123,6 +124,7 @@ def transcribe_video( ) -> tuple[list[Mapping[str, Any]], str]: import whisperx + settings = dialogue_config(config) cancellation.raise_if_cancelled() audio_name = hashlib.sha256( str(config.video_id).encode("utf-8") @@ -139,9 +141,12 @@ def transcribe_video( report_progress( progress, "preparing_transcription_model", - f"Preparing transcription model: WhisperX {config.whisper_model}.", + f"Preparing transcription model: WhisperX {settings.whisper_model}.", + ) + whisper_model = get_whisper_model( + settings.whisper_model, + config.device, ) - whisper_model = get_whisper_model(config.whisper_model, config.device) audio = whisperx.load_audio(str(audio_path)) report_progress( progress, @@ -150,7 +155,7 @@ def transcribe_video( ) transcription = whisper_model.transcribe( audio, - batch_size=config.transcription_batch_size, + batch_size=settings.transcription_batch_size, ) language = str(transcription["language"]) @@ -222,6 +227,7 @@ def index_dialogue( ) -> dict[str, Any]: if config.video_id is None: raise ValueError("IndexConfig.video_id is required for indexing.") + settings = dialogue_config(config) language = None if source.transcript is not None: @@ -241,7 +247,7 @@ def index_dialogue( phrases = build_dialogue_phrases( segments, - words_per_phrase=config.dialogue_words_per_phrase, + words_per_phrase=settings.words_per_phrase, ) if not phrases: return {"dialogue_phrases": 0, "language": language} @@ -249,11 +255,11 @@ def index_dialogue( report_progress( progress, "preparing_dialogue_model", - f"Preparing dialogue model: {config.sentence_model}.", + f"Preparing dialogue model: {settings.sentence_model}.", 0, len(phrases), ) - encoder = get_embedder(config.sentence_model, config.device) + encoder = get_embedder(settings.sentence_model, config.device) report_progress( progress, "dialogue_indexing", @@ -262,14 +268,14 @@ def index_dialogue( len(phrases), ) stored = 0 - for offset in range(0, len(phrases), config.dialogue_batch_size): + for offset in range(0, len(phrases), settings.embedding_batch_size): cancellation.raise_if_cancelled() - group = phrases[offset:offset + config.dialogue_batch_size] + group = phrases[offset:offset + settings.embedding_batch_size] vectors = encoder.encode( [phrase.text for phrase in group], batch_size=len(group), convert_to_numpy=True, - normalize_embeddings=config.normalize_dialogue_embeddings, + normalize_embeddings=settings.normalize_embeddings, ) stored += storage.upsert( "dialogue", diff --git a/src/vidxp/capabilities/dialogue/models.py b/src/vidxp/capabilities/dialogue/models.py new file mode 100644 index 0000000..449d50a --- /dev/null +++ b/src/vidxp/capabilities/dialogue/models.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from functools import lru_cache + + +@lru_cache +def get_embedder(model_name: str, device: str): + from sentence_transformers import SentenceTransformer + + return SentenceTransformer(model_name, device=device) + + +@lru_cache +def get_whisper_model(model_name: str, device: str): + import whisperx + + return whisperx.load_model(model_name, device, compute_type="float32") + + +@lru_cache +def get_alignment_model(language: str, device: str): + import whisperx + + return whisperx.load_align_model(language_code=language, device=device) + + +def clear_model_cache() -> None: + get_embedder.cache_clear() + get_whisper_model.cache_clear() + get_alignment_model.cache_clear() diff --git a/src/vidxp/capabilities/dialogue/operations.py b/src/vidxp/capabilities/dialogue/operations.py new file mode 100644 index 0000000..9fd98e3 --- /dev/null +++ b/src/vidxp/capabilities/dialogue/operations.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from vidxp.capabilities.contracts import ( + CapabilityContext, + CapabilityIndexResult, +) +from vidxp.capabilities.dialogue.config import dialogue_config +from vidxp.capabilities.dialogue.indexing import index_dialogue +from vidxp.capabilities.dialogue.models import get_embedder +from vidxp.capabilities.schemas import SearchInput, SearchResult +from vidxp.capabilities.search import search_embeddings +from vidxp.core.contracts import ( + CancellationToken, + IndexConfig, + VideoSource, +) +from vidxp.core.indexing_common import ProgressCallback +from vidxp.core.storage import IndexStorage + + +REQUIRED_METADATA = frozenset( + { + "dataset", + "split", + "run_id", + "video_id", + "source_id", + "start", + "end", + "text", + "phrase_id", + "modality", + } +) + + +def index_capability( + source: VideoSource, + *, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, + progress: ProgressCallback | None = None, + modalities: tuple[str, ...] = ("dialogue",), +) -> CapabilityIndexResult: + if modalities != ("dialogue",): + raise ValueError("The dialogue indexer only accepts dialogue.") + return CapabilityIndexResult( + summary=index_dialogue( + source, + config=config, + storage=storage, + cancellation=cancellation, + progress=progress, + ) + ) + + +def dialogue_embedding(query: str, config: IndexConfig) -> list[float]: + settings = dialogue_config(config) + encoder = get_embedder(settings.sentence_model, config.device) + encoded = encoder.encode( + [query], + convert_to_numpy=True, + normalize_embeddings=settings.normalize_embeddings, + ) + return encoded[0].tolist() + + +def search_dialogue( + query: str, + *, + config: IndexConfig, + top_k: int = 10, + video_id: str | None = None, + query_id: str | None = None, + filters: Mapping[str, Any] | None = None, + storage: IndexStorage | None = None, +) -> SearchResult: + cleaned = query.strip() + if not cleaned: + raise ValueError("Search query must not be empty.") + if top_k <= 0: + raise ValueError("top_k must be greater than zero.") + return search_embeddings( + cleaned, + "dialogue", + dialogue_embedding(cleaned, config), + config=config, + required_metadata=REQUIRED_METADATA, + top_k=top_k, + video_id=video_id, + query_id=query_id, + filters=filters, + storage=storage, + ) + + +def search_operation( + context: CapabilityContext, + request: SearchInput, +) -> SearchResult: + config = context.require_config() + return search_dialogue( + request.query, + config=config, + top_k=request.top_k, + video_id=config.video_id, + ) diff --git a/src/vidxp/capabilities/dialogue/requirements.txt b/src/vidxp/capabilities/dialogue/requirements.txt new file mode 100644 index 0000000..c97c911 --- /dev/null +++ b/src/vidxp/capabilities/dialogue/requirements.txt @@ -0,0 +1,3 @@ +moviepy==1.0.3 +sentence-transformers>=3.4,<4 +whisperx>=3.8.6,<3.9 diff --git a/src/vidxp/capabilities/registry.py b/src/vidxp/capabilities/registry.py new file mode 100644 index 0000000..2a7a633 --- /dev/null +++ b/src/vidxp/capabilities/registry.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from types import MappingProxyType +from typing import Any, Iterable, Mapping + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +from vidxp.capabilities.actor.definition import DEFINITION as ACTOR +from vidxp.capabilities.contracts import ( + CapabilityDefinition, + RuntimeCheck, + capability_install_hint, +) +from vidxp.capabilities.dialogue.definition import DEFINITION as DIALOGUE +from vidxp.capabilities.scene.definition import DEFINITION as SCENE +from vidxp.core.contracts import VideoSource +from vidxp.dependencies import ( + active_requirements, + inspect_requirement, + installed_base_requirements, + packaged_requirements, +) + + +_BUILT_INS = (DIALOGUE, SCENE, ACTOR) +CAPABILITIES = MappingProxyType( + {capability.name: capability for capability in _BUILT_INS} +) + +if len(CAPABILITIES) != len(_BUILT_INS): + raise RuntimeError("Capability names must be unique.") + + +def capability_names() -> tuple[str, ...]: + return tuple(CAPABILITIES) + + +def index_capability_names() -> tuple[str, ...]: + return tuple( + name + for name, capability in CAPABILITIES.items() + if capability.indexer is not None + ) + + +def preparable_capability_names() -> tuple[str, ...]: + return tuple( + name + for name, capability in CAPABILITIES.items() + if capability.prepare is not None + ) + + +def get_capability(name: str) -> CapabilityDefinition: + try: + return CAPABILITIES[name] + except KeyError as exc: + available = ", ".join(capability_names()) + raise ValueError( + f"Unknown capability {name!r}. Available capabilities: {available}." + ) from exc + + +def validate_capability_names(names: Iterable[str]) -> tuple[str, ...]: + selected = tuple(dict.fromkeys(str(name).strip() for name in names)) + if not selected: + raise ValueError("At least one capability is required.") + for name in selected: + get_capability(name) + return selected + + +def collection_names( + names: Iterable[str] | None = None, +) -> dict[str, str]: + selected = ( + index_capability_names() + if names is None + else validate_capability_names(names) + ) + return { + name: get_capability(name).collection_name + for name in selected + if get_capability(name).indexer is not None + } + + +def validate_capability_options( + names: Iterable[str], + options: Mapping[str, Mapping[str, Any]] | None, +) -> dict[str, dict[str, Any]]: + selected = validate_capability_names(names) + supplied = dict(options or {}) + unknown = sorted(set(supplied) - set(selected)) + if unknown: + raise ValueError( + "Options were supplied for disabled capabilities: " + + ", ".join(unknown) + ) + return { + name: get_capability(name) + .config_model.model_validate(supplied.get(name, {})) + .model_dump(mode="python") + for name in selected + } + + +def requirements_for( + names: Iterable[str], + *, + source: VideoSource | None = None, +) -> tuple[Requirement, ...]: + selected = tuple( + get_capability(name) + for name in validate_capability_names(names) + ) + requirements = list( + active_requirements( + packaged_requirements( + "vidxp", + "requirements/storage.txt", + ) + ) + if any(capability.indexer is not None for capability in selected) + else () + ) + for capability in selected: + capability_requirements = active_requirements( + packaged_requirements( + f"vidxp.capabilities.{capability.name}" + ) + ) + requirements.extend( + capability_requirements + if source is None + else capability.source_requirements( + source, + capability_requirements, + ) + ) + unique = {str(requirement): requirement for requirement in requirements} + return tuple(unique.values()) + + +def runtime_checks_for( + names: Iterable[str], + *, + source: VideoSource | None = None, +) -> tuple[RuntimeCheck, ...]: + checks = ( + check + for name in validate_capability_names(names) + for check in get_capability(name).runtime_checks + if check.applies(source) + ) + return tuple({check.label: check for check in checks}.values()) + + +def dependency_checks(names: Iterable[str]) -> tuple[dict, ...]: + return tuple( + inspect_requirement(requirement) + for requirement in requirements_for(names) + ) + tuple( + check.inspect() + for check in runtime_checks_for(names) + ) + + +def require_dependencies( + names: Iterable[str], + *, + source: VideoSource, +) -> None: + selected = validate_capability_names(names) + failures = [ + result + for requirement in requirements_for(selected, source=source) + if not (result := inspect_requirement(requirement))["ok"] + ] + failures.extend( + result + for check in runtime_checks_for(selected, source=source) + if not (result := check.inspect())["ok"] + ) + if failures: + details = "; ".join( + f"{failure['name']}: {failure['error']}" + for failure in failures + ) + extras = ",".join( + get_capability(name).extra for name in selected + ) + raise RuntimeError( + f"Capability dependencies are unavailable: {details}. " + + capability_install_hint(extras) + ) + + +def runtime_distributions() -> tuple[str, ...]: + distributions = { + canonicalize_name(requirement.name) + for requirement in installed_base_requirements() + } + distributions.update( + canonicalize_name(requirement.name) + for requirement in requirements_for(capability_names()) + ) + return tuple(sorted(distributions, key=str.lower)) diff --git a/src/vidxp/capabilities/scene/__init__.py b/src/vidxp/capabilities/scene/__init__.py new file mode 100644 index 0000000..6f5e85b --- /dev/null +++ b/src/vidxp/capabilities/scene/__init__.py @@ -0,0 +1 @@ +"""Scene capability implementation.""" diff --git a/src/vidxp/capabilities/scene/config.py b/src/vidxp/capabilities/scene/config.py new file mode 100644 index 0000000..87b2e34 --- /dev/null +++ b/src/vidxp/capabilities/scene/config.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pydantic import Field + +from vidxp.capabilities.contracts import CapabilityConfig +from vidxp.core.contracts import IndexConfig + + +class SceneConfig(CapabilityConfig): + batch_size: int = Field(default=32, gt=0) + model: str = Field(default="ViT-B/32", min_length=1) + + +def scene_config(config: IndexConfig) -> SceneConfig: + return SceneConfig.model_validate(config.options_for("scene")) diff --git a/src/vidxp/capabilities/scene/definition.py b/src/vidxp/capabilities/scene/definition.py new file mode 100644 index 0000000..6deaf99 --- /dev/null +++ b/src/vidxp/capabilities/scene/definition.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from vidxp.capabilities.contracts import ( + CapabilityDefinition, + OperationDefinition, + PreparationContext, +) +from vidxp.capabilities.scene.config import SceneConfig, scene_config +from vidxp.capabilities.scene.indexing import VISUAL_PROCESSOR +from vidxp.capabilities.scene.models import get_clip_model +from vidxp.capabilities.scene.operations import search_operation +from vidxp.capabilities.schemas import SearchInput, SearchResult +from vidxp.capabilities.visual import index_capabilities +from vidxp.core.contracts import IndexConfig, VideoSource +from vidxp.core.indexing_common import ProgressCallback + +def prepare_models( + context: PreparationContext, + progress: ProgressCallback | None, +) -> tuple[str, ...]: + settings = SceneConfig.model_validate(context.settings) + if progress is not None: + progress( + { + "state": "preparing", + "stage": "scene_model", + "message": ( + f"Preparing scene model: CLIP {settings.model}" + ), + } + ) + get_clip_model(settings.model, context.device) + return (settings.model,) + + +def model_manifest( + config: IndexConfig, + _sources: tuple[VideoSource, ...], +) -> Mapping[str, Any]: + return {"scene": scene_config(config).model} + + +DEFINITION = CapabilityDefinition( + name="scene", + description="Index and search visual scenes.", + extra="scene", + config_model=SceneConfig, + collection_name="scene", + indexer=index_capabilities, + index_processor=VISUAL_PROCESSOR, + index_stage="visual_indexing", + prepare=prepare_models, + model_manifest=model_manifest, + operations={ + "search": OperationDefinition( + input_model=SearchInput, + output_model=SearchResult, + handler=search_operation, + ) + }, +) diff --git a/src/vidxp/capabilities/scene/indexing.py b/src/vidxp/capabilities/scene/indexing.py new file mode 100644 index 0000000..33cb1a0 --- /dev/null +++ b/src/vidxp/capabilities/scene/indexing.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from vidxp.capabilities.scene.config import scene_config +from vidxp.capabilities.scene.models import get_clip_model +from vidxp.core.contracts import ( + CancellationToken, + IndexConfig, + StorageRecord, + batched, + stable_source_id, +) +from vidxp.core.indexing_common import ProgressCallback, report_progress +from vidxp.core.storage import IndexStorage + + +@dataclass +class SceneIndexState: + model: Any + preprocess: Any + stored_frames: int = 0 + + +def encode_scene_batch(samples, model, preprocess, device): + import torch + from PIL import Image + + images = torch.stack( + [preprocess(Image.fromarray(sample.frame)) for sample in samples] + ).to(device) + with torch.no_grad(): + features = model.encode_image(images) + features /= features.norm(dim=-1, keepdim=True) + return features.cpu().numpy().tolist() + + +def scene_records( + samples, + vectors, + info, + config: IndexConfig, +) -> list[StorageRecord]: + records = [] + for sample, vector in zip(samples, vectors): + end = min( + info.duration, + sample.timestamp + config.frame_stride / info.fps, + ) + if end <= sample.timestamp: + end = sample.timestamp + 1 / info.fps + source_id = stable_source_id( + config.run_id, + str(config.video_id), + "scene", + f"f{sample.frame_index:012d}", + ) + records.append( + StorageRecord( + source_id=source_id, + embedding=vector, + metadata={ + **config.record_identity("scene", source_id), + "frame_index": sample.frame_index, + "timestamp": sample.timestamp, + "start": sample.timestamp, + "end": end, + "fps": info.fps, + "duration": info.duration, + }, + ) + ) + return records + + +def process_scene_samples( + samples, + *, + state: SceneIndexState, + info, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, +) -> None: + settings = scene_config(config) + for group in batched(samples, settings.batch_size): + cancellation.raise_if_cancelled() + vectors = encode_scene_batch( + group, + state.model, + state.preprocess, + config.device, + ) + state.stored_frames += storage.upsert( + "scene", + scene_records(group, vectors, info, config), + batch_size=config.storage_batch_size, + cancellation=cancellation, + ) + + +class SceneVisualProcessor: + def batch_size(self, config: IndexConfig) -> int: + return scene_config(config).batch_size + + def prepare( + self, + config: IndexConfig, + progress: ProgressCallback | None, + ) -> SceneIndexState: + settings = scene_config(config) + report_progress( + progress, + "preparing_scene_model", + f"Preparing scene model: CLIP {settings.model}.", + ) + model, preprocess = get_clip_model(settings.model, config.device) + return SceneIndexState(model, preprocess) + + def process( + self, + samples, + *, + state: SceneIndexState, + info, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, + ) -> None: + process_scene_samples( + samples, + state=state, + info=info, + config=config, + storage=storage, + cancellation=cancellation, + ) + + def finalize( + self, + state: SceneIndexState, + *, + config: IndexConfig, + storage: IndexStorage, + ) -> tuple[dict[str, Any], int]: + return {"scene_frames": state.stored_frames}, state.stored_frames + + +VISUAL_PROCESSOR = SceneVisualProcessor() diff --git a/src/vidxp/capabilities/scene/models.py b/src/vidxp/capabilities/scene/models.py new file mode 100644 index 0000000..380f9f1 --- /dev/null +++ b/src/vidxp/capabilities/scene/models.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from functools import lru_cache + + +@lru_cache +def get_clip_model(model_name: str, device: str): + import clip + + return clip.load(model_name, device=device) + + +def clear_model_cache() -> None: + get_clip_model.cache_clear() diff --git a/src/vidxp/capabilities/scene/operations.py b/src/vidxp/capabilities/scene/operations.py new file mode 100644 index 0000000..be04be7 --- /dev/null +++ b/src/vidxp/capabilities/scene/operations.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from vidxp.capabilities.contracts import CapabilityContext +from vidxp.capabilities.scene.config import scene_config +from vidxp.capabilities.scene.models import get_clip_model +from vidxp.capabilities.schemas import SearchInput, SearchResult +from vidxp.capabilities.search import search_embeddings +from vidxp.core.contracts import IndexConfig +from vidxp.core.storage import IndexStorage + + +REQUIRED_METADATA = frozenset( + { + "dataset", + "split", + "run_id", + "video_id", + "source_id", + "start", + "end", + "frame_index", + "timestamp", + "fps", + "duration", + "modality", + } +) + + +def scene_embedding(query: str, config: IndexConfig) -> list[float]: + import clip + import torch + + settings = scene_config(config) + model, _ = get_clip_model(settings.model, config.device) + tokens = clip.tokenize([query]).to(config.device) + with torch.no_grad(): + features = model.encode_text(tokens) + features /= features.norm(dim=-1, keepdim=True) + return features.cpu().numpy().tolist()[0] + + +def search_scene( + query: str, + *, + config: IndexConfig, + top_k: int = 10, + video_id: str | None = None, + query_id: str | None = None, + filters: Mapping[str, Any] | None = None, + storage: IndexStorage | None = None, +) -> SearchResult: + cleaned = query.strip() + if not cleaned: + raise ValueError("Search query must not be empty.") + if top_k <= 0: + raise ValueError("top_k must be greater than zero.") + return search_embeddings( + cleaned, + "scene", + scene_embedding(cleaned, config), + config=config, + required_metadata=REQUIRED_METADATA, + top_k=top_k, + video_id=video_id, + query_id=query_id, + filters=filters, + storage=storage, + ) + + +def search_operation( + context: CapabilityContext, + request: SearchInput, +) -> SearchResult: + config = context.require_config() + return search_scene( + request.query, + config=config, + top_k=request.top_k, + video_id=config.video_id, + ) diff --git a/src/vidxp/capabilities/scene/requirements.txt b/src/vidxp/capabilities/scene/requirements.txt new file mode 100644 index 0000000..213eddc --- /dev/null +++ b/src/vidxp/capabilities/scene/requirements.txt @@ -0,0 +1,7 @@ +clip-anytorch==2.6.0 +numpy>=2.1,<3 +opencv-python +Pillow>=7.0.0 +# clip-anytorch still imports pkg_resources, removed in setuptools 81. +setuptools<81 +torch diff --git a/src/vidxp/capabilities/schemas.py b/src/vidxp/capabilities/schemas.py new file mode 100644 index 0000000..a70b948 --- /dev/null +++ b/src/vidxp/capabilities/schemas.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import Field + +from vidxp.capabilities.contracts import CapabilityInput, CapabilityOutput +from vidxp.core.contracts import INDEX_SCHEMA_VERSION + + +class SearchInput(CapabilityInput): + query: str = Field(min_length=1) + top_k: int = Field(default=10, gt=0) + + +class SearchHit(CapabilityOutput): + rank: int = Field(gt=0) + video_id: str = Field(min_length=1) + start: float = Field(ge=0) + end: float = Field(gt=0) + score: float + raw_distance: float + modality: str = Field(min_length=1) + source_id: str = Field(min_length=1) + metadata: dict[str, Any] = Field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return self.model_dump(mode="json") + + +class SearchResult(CapabilityOutput): + query_id: str = Field(min_length=1) + query: str = Field(min_length=1) + modality: str = Field(min_length=1) + hits: tuple[SearchHit, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": INDEX_SCHEMA_VERSION, + **self.model_dump(mode="json"), + } + + def to_prediction(self) -> dict[str, list[dict[str, Any]]]: + return { + self.query_id: [ + hit.model_dump(mode="json") for hit in self.hits + ] + } diff --git a/src/vidxp/core/search.py b/src/vidxp/capabilities/search.py similarity index 68% rename from src/vidxp/core/search.py rename to src/vidxp/capabilities/search.py index ab9053e..4b8a6ff 100644 --- a/src/vidxp/core/search.py +++ b/src/vidxp/capabilities/search.py @@ -6,46 +6,14 @@ from pathlib import Path from typing import Any, Mapping +from vidxp.capabilities.schemas import SearchHit, SearchResult from vidxp.core.contracts import ( IndexConfig, IndexSchemaError, - SearchHit, - SearchResult, ) -from vidxp.core.models import get_clip_model, get_embedder from vidxp.core.storage import IndexStorage -REQUIRED_METADATA = { - "dialogue": { - "dataset", - "split", - "run_id", - "video_id", - "source_id", - "start", - "end", - "text", - "phrase_id", - "modality", - }, - "scene": { - "dataset", - "split", - "run_id", - "video_id", - "source_id", - "start", - "end", - "frame_index", - "timestamp", - "fps", - "duration", - "modality", - }, -} - - def distance_to_score(raw_distance: float) -> float: """Map distance to an ordering score without claiming probability. @@ -77,33 +45,11 @@ def stable_query_id( return f"{modality}:{digest}" -def _dialogue_embedding(query: str, config: IndexConfig) -> list[float]: - encoder = get_embedder(config.sentence_model, config.device) - encoded = encoder.encode( - [query], - convert_to_numpy=True, - normalize_embeddings=config.normalize_dialogue_embeddings, - ) - return encoded[0].tolist() - - -def _scene_embedding(query: str, config: IndexConfig) -> list[float]: - import clip - import torch - - model, _ = get_clip_model(config.clip_model, config.device) - tokens = clip.tokenize([query]).to(config.device) - with torch.no_grad(): - features = model.encode_text(tokens) - features /= features.norm(dim=-1, keepdim=True) - return features.cpu().numpy().tolist()[0] - - def _to_hits( modality: str, rows: list[dict[str, Any]], + required_metadata: frozenset[str], ) -> tuple[SearchHit, ...]: - required = REQUIRED_METADATA[modality] ordered = sorted( rows, key=lambda row: (row["raw_distance"], row["source_id"]), @@ -111,7 +57,7 @@ def _to_hits( hits = [] for rank, row in enumerate(ordered, start=1): metadata = row["metadata"] - missing = sorted(required - metadata.keys()) + missing = sorted(required_metadata - metadata.keys()) if missing: raise IndexSchemaError( "The saved index predates the benchmark-ready schema and must " @@ -141,11 +87,13 @@ def _to_hits( return tuple(hits) -def search( +def search_embeddings( query: str, modality: str, + embedding: list[float], *, config: IndexConfig, + required_metadata: frozenset[str], top_k: int = 10, video_id: str | None = None, query_id: str | None = None, @@ -161,13 +109,6 @@ def search( raise ValueError( f"The {modality} modality is not present in this index run." ) - if modality == "dialogue": - embedding = _dialogue_embedding(query, config) - elif modality == "scene": - embedding = _scene_embedding(query, config) - else: - raise ValueError("Semantic search supports dialogue and scene modalities.") - owns_storage = storage is None store = storage or IndexStorage(config) try: @@ -185,18 +126,10 @@ def search( query_id=query_id or stable_query_id(query, modality, config), query=query, modality=modality, - hits=_to_hits(modality, rows), + hits=_to_hits(modality, rows, required_metadata), ) -def search_dialogue(query: str, **options: Any) -> SearchResult: - return search(query, "dialogue", **options) - - -def search_scene(query: str, **options: Any) -> SearchResult: - return search(query, "scene", **options) - - def serialize_predictions( results: list[SearchResult], path: str | Path | None = None, diff --git a/src/vidxp/capabilities/visual.py b/src/vidxp/capabilities/visual.py new file mode 100644 index 0000000..b7ef539 --- /dev/null +++ b/src/vidxp/capabilities/visual.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +from dataclasses import dataclass +from time import perf_counter +from typing import Any, Protocol, Sequence + +from vidxp.capabilities.contracts import CapabilityIndexResult +from vidxp.core.contracts import ( + CancellationToken, + IndexConfig, + VideoSource, +) +from vidxp.core.indexing_common import ProgressCallback, report_progress +from vidxp.core.storage import IndexStorage +from vidxp.core.video import ( + FrameSample, + FrameStreamStats, + iter_frame_batches, + probe_video, +) + + +class VisualProcessor(Protocol): + def batch_size(self, config: IndexConfig) -> int: ... + + def prepare( + self, + config: IndexConfig, + progress: ProgressCallback | None, + ) -> Any: ... + + def process( + self, + samples: Sequence[FrameSample], + *, + state: Any, + info: Any, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, + ) -> None: ... + + def finalize( + self, + state: Any, + *, + config: IndexConfig, + storage: IndexStorage, + ) -> tuple[dict[str, Any], int]: ... + + +@dataclass +class _Participant: + name: str + processor: VisualProcessor + state: Any + + +def _rgb_samples(samples) -> list[FrameSample]: + import cv2 + + return [ + FrameSample( + frame_index=sample.frame_index, + timestamp=sample.timestamp, + frame=cv2.cvtColor(sample.frame, cv2.COLOR_BGR2RGB), + ) + for sample in samples + ] + + +def _participants( + names: Sequence[str], + *, + config: IndexConfig, + progress: ProgressCallback | None, + timings: dict[str, float], +) -> list[_Participant]: + from vidxp.capabilities.registry import get_capability + + participants = [] + for name in names: + processor = get_capability(name).index_processor + if processor is None: + raise ValueError( + f"Capability {name!r} does not provide a visual processor." + ) + started = perf_counter() + state = processor.prepare(config, progress) + timings[name] = perf_counter() - started + participants.append(_Participant(name, processor, state)) + return participants + + +def _consume_visual_stream( + source: VideoSource, + *, + participants: Sequence[_Participant], + expected: int, + info: Any, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, + progress: ProgressCallback | None, + timings: dict[str, float], +) -> FrameStreamStats: + stream_stats = FrameStreamStats() + stream = iter( + iter_frame_batches( + source.path, + frame_stride=config.frame_stride, + batch_size=max( + participant.processor.batch_size(config) + for participant in participants + ), + cancellation=cancellation, + stats=stream_stats, + ) + ) + while True: + stream_started = perf_counter() + try: + samples = next(stream) + except StopIteration: + timings["frame_stream"] += perf_counter() - stream_started + break + rgb_samples = _rgb_samples(samples) + timings["frame_stream"] += perf_counter() - stream_started + + for participant in participants: + processor_started = perf_counter() + participant.processor.process( + rgb_samples, + state=participant.state, + info=info, + config=config, + storage=storage, + cancellation=cancellation, + ) + timings[participant.name] += ( + perf_counter() - processor_started + ) + + report_progress( + progress, + "visual_indexing", + "Indexing the shared sampled-frame stream.", + stream_stats.frames_materialized, + expected, + ) + return stream_stats + + +def _finalize( + participants: Sequence[_Participant], + *, + config: IndexConfig, + storage: IndexStorage, + timings: dict[str, float], +) -> tuple[dict[str, Any], int]: + summary: dict[str, Any] = {} + frame_operations = 0 + for participant in participants: + started = perf_counter() + result, operations = participant.processor.finalize( + participant.state, + config=config, + storage=storage, + ) + timings[participant.name] += perf_counter() - started + duplicate = set(summary).intersection(result) + if duplicate: + raise ValueError( + "Visual capability summaries contain duplicate keys: " + + ", ".join(sorted(duplicate)) + ) + summary.update(result) + frame_operations += operations + return summary, frame_operations + + +def index_visuals( + source: VideoSource, + *, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, + progress: ProgressCallback | None = None, + modalities: Sequence[str] | None = None, +) -> CapabilityIndexResult: + if config.video_id is None: + raise ValueError("IndexConfig.video_id is required for indexing.") + if source.path is None: + raise ValueError("Visual indexing requires a video path.") + + selected = tuple( + config.enabled_modalities if modalities is None else modalities + ) + if not selected: + raise ValueError("At least one visual capability must be selected.") + + started = perf_counter() + info = probe_video(source.path) + expected = ( + info.frame_count + config.frame_stride - 1 + ) // config.frame_stride + timings = { + "frame_stream": 0.0, + } + participants = _participants( + selected, + config=config, + progress=progress, + timings=timings, + ) + report_progress( + progress, + "visual_indexing", + "Decoding sampled frames for " + + " and ".join(selected) + + " indexing.", + 0, + expected, + ) + stream_stats = _consume_visual_stream( + source, + participants=participants, + expected=expected, + info=info, + config=config, + storage=storage, + cancellation=cancellation, + progress=progress, + timings=timings, + ) + capability_summary, frame_operations = _finalize( + participants, + config=config, + storage=storage, + timings=timings, + ) + sampled_frames = stream_stats.frames_materialized + timings["visual_total"] = perf_counter() - started + return CapabilityIndexResult( + summary={ + "source_frames_advanced": stream_stats.frames_advanced, + "sampled_frames": sampled_frames, + "processed_frames": sampled_frames, + "frame_operations": frame_operations, + "duration": info.duration, + "fps": info.fps, + **capability_summary, + }, + timings=timings, + ) + + +def index_capabilities( + source: VideoSource, + *, + config: IndexConfig, + storage: IndexStorage, + cancellation: CancellationToken, + progress: ProgressCallback | None = None, + modalities: Sequence[str] | None = None, +) -> CapabilityIndexResult: + return index_visuals( + source, + config=config, + storage=storage, + cancellation=cancellation, + progress=progress, + modalities=modalities, + ) diff --git a/src/vidxp/cli.py b/src/vidxp/cli.py index 958b265..7135302 100644 --- a/src/vidxp/cli.py +++ b/src/vidxp/cli.py @@ -1,50 +1,70 @@ from __future__ import annotations +import json +import os +import sys +from pathlib import Path from typing import Annotated import typer -from rich import print from vidxp import __version__ -from vidxp.benchmarks.cli import app as benchmark_app -from vidxp.core.actor_results import ( - ActorClusterNotFoundError, - render_actor_result, -) -from vidxp.core.contracts import IndexConfig, IndexSchemaError -from vidxp.core.models import ( - INDEXING_DEPENDENCIES, - dependency_failures, - get_alignment_model, - get_clip_model, - get_embedder, - get_whisper_model, -) -from vidxp.core.runner import ( - index_video, - local_config_from_status, -) -from vidxp.core.search import search_dialogue, search_scene -from vidxp.core.video import ffmpeg_binary +from vidxp.application import VidXPService +from vidxp.capabilities.actor.results import ActorClusterNotFoundError +from vidxp.capabilities.registry import CAPABILITIES +from vidxp.cli_commands.index import app as index_app +from vidxp.cli_commands.repositories import app as repositories_app +from vidxp.cli_commands.runtime import doctor, prepare, ui +from vidxp.cli_commands.search import app as search_app +from vidxp.cli_support import CLIState, OutputFormat +from vidxp.core.contracts import IndexSchemaError +from vidxp.dependencies import requirements_available from vidxp.index_state import ( IndexingInProgressError, IndexNotReadyError, - require_ready_index, ) +from vidxp.repositories import resolve_repository -app = typer.Typer(no_args_is_help=True) -app.add_typer(benchmark_app, name="benchmark") +app = typer.Typer( + no_args_is_help=True, + help="Index and search video with installable capabilities.", +) +app.add_typer(index_app, name="index") +app.add_typer(search_app, name="search") +app.add_typer(repositories_app, name="repositories") + + +def _load_benchmark_app(): + if not requirements_available("vidxp.benchmarks"): + return None + from vidxp.benchmarks.cli import app as benchmark_app + + return benchmark_app + + +if _benchmark_app := _load_benchmark_app(): + app.add_typer(_benchmark_app, name="benchmark") +for _capability in CAPABILITIES.values(): + if _capability.cli_factory is not None: + app.add_typer( + _capability.cli_factory(), + name=_capability.cli_name, + ) +app.command()(doctor) +app.command()(prepare) +app.command()(ui) def _show_version(value: bool) -> None: if value: - print(f"VidXP {__version__}") + typer.echo(f"VidXP {__version__}") raise typer.Exit() @app.callback() def app_options( + ctx: typer.Context, version: Annotated[ bool, typer.Option( @@ -55,264 +75,143 @@ def app_options( help="Show the installed VidXP version and exit.", ), ] = False, -) -> None: - """Index and search video by dialogue, scene, and actor.""" - - -def _modalities(value: str) -> tuple[str, ...]: - modalities = tuple( - item.strip().lower() - for item in value.split(",") - if item.strip() - ) - try: - IndexConfig(enabled_modalities=modalities) - except ValueError as exc: - raise typer.BadParameter(str(exc)) from exc - return modalities - - -def _active_config() -> tuple[IndexConfig, dict]: - status = require_ready_index() - try: - return local_config_from_status(status), status - except IndexSchemaError as exc: - raise IndexNotReadyError(str(exc)) from exc - - -def _require_modality(config: IndexConfig, modality: str) -> None: - if modality not in config.enabled_modalities: - raise IndexNotReadyError( - f"The {modality} modality is not present in this index." - ) - - -@app.command() -def videoindex( - path: str, - modalities: Annotated[ - str, + repository_name: Annotated[ + str | None, typer.Option( - "--modalities", - "-m", - help="Comma-separated dialogue, scene, and actor modalities.", + "--repository", + "-r", + help="Named repository to use.", ), - ] = "dialogue,scene,actor", - frame_stride: Annotated[ - int, - typer.Option( - "--frame-stride", - min=1, - help=( - "Materialize every Nth frame for scene and actor modalities." - ), - ), - ] = 1, -): - """Index one local video, replacing the previous local index.""" - - config = IndexConfig.local( - enabled_modalities=_modalities(modalities), - frame_stride=frame_stride, - ) - last_stage = None - last_percent = None - - def progress(event): - nonlocal last_percent, last_stage - stage = event["stage"] - current, total = event.get("current"), event.get("total") - percent = ( - int(current * 100 / total) - if current is not None and total - else None - ) - if stage != last_stage: - print(f"[cyan]{event['message']}[/cyan]") - last_stage = stage - last_percent = percent - elif percent is not None and ( - last_percent is None or percent >= last_percent + 10 - ): - print(f"[cyan]{event['message']} {percent}%[/cyan]") - last_percent = percent - - summary = index_video(path, progress_callback=progress, config=config) - print("[bold green]Video indexing completed successfully.[/bold green]") - return summary - - -@app.command() -def doctor( - modalities: Annotated[ - str, + ] = None, + config_file: Annotated[ + Path | None, typer.Option( - "--modalities", - "-m", - help="Only validate dependencies for these modalities.", + "--config", + dir_okay=False, + help="Repository configuration file.", ), - ] = "dialogue,scene,actor", -): - """Validate selected indexing dependencies without downloading models.""" - - selected = _modalities(modalities) - failures = dict( - dependency_failures( - selected, - needs_transcription="dialogue" in selected, - ) - ) - checked_labels = [] - for modality in selected: - for dependency in INDEXING_DEPENDENCIES[modality]: - if dependency.label not in checked_labels: - checked_labels.append(dependency.label) - if "dialogue" in selected: - for dependency in INDEXING_DEPENDENCIES["transcription"]: - if dependency.label not in checked_labels: - checked_labels.append(dependency.label) - - for label in checked_labels: - if label in failures: - print(f"[bold red]FAILED[/bold red] {label}: {failures[label]}") - else: - print(f"[green]OK[/green] {label}") - - if "dialogue" in selected: - try: - resolved_ffmpeg = ffmpeg_binary() - print(f"[green]OK[/green] FFmpeg: {resolved_ffmpeg}") - except Exception as exc: - failures["FFmpeg"] = f"{type(exc).__name__}: {exc}" - print(f"[bold red]FAILED[/bold red] FFmpeg: {failures['FFmpeg']}") - - if failures: - raise typer.Exit(1) - print("[bold green]Selected VidXP dependencies are available.[/bold green]") - - -@app.command() -def prepare( - modalities: Annotated[ - str, + ] = None, + index_directory: Annotated[ + Path | None, typer.Option( - "--modalities", - "-m", - help="Only prepare models for these modalities.", + "--index-dir", + file_okay=False, + help="Override the selected repository index directory.", ), - ] = "dialogue,scene", - language: Annotated[ + ] = None, + device: Annotated[ str | None, typer.Option( - "--language", - "-l", - help="Also cache the WhisperX alignment model for this language.", + "--device", + help="Override the selected repository runtime device.", ), ] = None, -): - """Download and cache selected runtime models before indexing.""" - - selected = _modalities(modalities) - config = IndexConfig.local(enabled_modalities=selected) - try: - failures = dependency_failures( - selected, - needs_transcription="dialogue" in selected, - ) - if failures: - details = "; ".join( - f"{label}: {error}" for label, error in failures - ) - raise RuntimeError(details) - if "dialogue" in selected: - print(f"[cyan]Preparing dialogue model: {config.sentence_model}[/cyan]") - get_embedder(config.sentence_model, config.device) - print( - f"[cyan]Preparing transcription model: " - f"WhisperX {config.whisper_model}[/cyan]" - ) - get_whisper_model(config.whisper_model, config.device) - if language: - print(f"[cyan]Preparing the {language} alignment model.[/cyan]") - get_alignment_model(language, config.device) - if "scene" in selected: - print(f"[cyan]Preparing scene model: CLIP {config.clip_model}[/cyan]") - get_clip_model(config.clip_model, config.device) - except Exception as exc: - print( - f"[bold red]Model preparation failed: " - f"{type(exc).__name__}: {exc}[/bold red]" - ) - raise typer.Exit(1) from exc - print("[bold green]Selected VidXP runtime models are prepared.[/bold green]") - - -@app.command() -def dialogue(query: str): - config, _ = _active_config() - _require_modality(config, "dialogue") - print("[green]Searching dialogue...[/green]") - result = search_dialogue( - query, - config=config, - top_k=1, - video_id=config.video_id, + output_format: Annotated[ + OutputFormat, + typer.Option( + "--format", + envvar="VIDXP_OUTPUT_FORMAT", + help="Default command output format.", + ), + ] = OutputFormat.rich, + quiet: Annotated[ + bool, + typer.Option("--quiet", "-q", help="Suppress progress output."), + ] = False, +) -> None: + registry, repository = resolve_repository( + registry_path=config_file, + name=repository_name, + index_directory=index_directory, + device=device, ) - if not result.hits: - raise IndexNotReadyError( - "The completed index contains no searchable dialogue phrases." - ) - print("[green]Dialogue found !!![/green]") - timestamp = result.hits[0].start - print(f"[bold green]{timestamp:.3f} seconds[/bold green]") - return timestamp - - -@app.command() -def scene(query: str): - config, _ = _active_config() - _require_modality(config, "scene") - print("[green]Searching scene...[/green]") - result = search_scene( - query, - config=config, - top_k=1, - video_id=config.video_id, + ctx.obj = CLIState( + service=VidXPService( + repository.index_directory, + device=repository.device, + ), + registry=registry, + repository=repository, + output_format=output_format, + quiet=quiet, ) - if not result.hits: - raise IndexNotReadyError( - "The completed index contains no searchable scene frames." - ) - print("[green]Scene found...[/green]") - timestamp = result.hits[0].start - print(f"[bold green]{timestamp:.3f} seconds[/bold green]") - return timestamp -@app.command() -def actor(cluster_id: str, input_path: str, output_path: str = "output.mp4"): - config, _ = _active_config() - _require_modality(config, "actor") - try: - render_actor_result( - config, - cluster_id, - input_path, - output_path, +def _wants_json(arguments: list[str] | None = None) -> bool: + values = list(sys.argv[1:] if arguments is None else arguments) + if "--json" in values: + return True + for index, value in enumerate(values): + if value == "--format" and index + 1 < len(values): + return values[index + 1].lower() == "json" + if value.startswith("--format="): + return value.split("=", 1)[1].lower() == "json" + return os.environ.get("VIDXP_OUTPUT_FORMAT", "").lower() == "json" + + +def _error_message(exc: Exception) -> str: + formatter = getattr(exc, "format_message", None) + return str(formatter()) if formatter is not None else str(exc) + + +def _exit_code(exc: Exception) -> int: + return int(getattr(exc, "exit_code", 1) or 1) + + +def _emit_error(exc: Exception, *, json_output: bool) -> None: + message = _error_message(exc) + if json_output: + typer.echo( + json.dumps( + { + "ok": False, + "error": { + "type": type(exc).__name__, + "message": message, + "exit_code": _exit_code(exc), + }, + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ), + err=True, ) - except ActorClusterNotFoundError as exc: - raise IndexNotReadyError( - str(exc) - ) from exc - print(f"[green]Video saved as {output_path}[/green]") + elif show := getattr(exc, "show", None): + show(file=sys.stderr) + else: + typer.secho(message, fg=typer.colors.RED, err=True) -def main(): +def main() -> None: try: - app() - except (IndexNotReadyError, IndexingInProgressError) as exc: - print(f"[bold red]{exc}[/bold red]") + app(standalone_mode=False) + except typer.Exit as exc: + raise SystemExit(exc.exit_code) from None + except typer.Abort as exc: + _emit_error(exc, json_output=_wants_json()) raise SystemExit(1) from exc + except Exception as exc: + is_command_error = hasattr(exc, "exit_code") and hasattr( + exc, + "format_message", + ) + is_expected_runtime_error = isinstance( + exc, + ( + ActorClusterNotFoundError, + FileNotFoundError, + IndexNotReadyError, + IndexingInProgressError, + IndexSchemaError, + RuntimeError, + ValueError, + ), + ) + if not is_command_error and not is_expected_runtime_error: + raise + _emit_error(exc, json_output=_wants_json()) + raise SystemExit(_exit_code(exc)) from exc if __name__ == "__main__": diff --git a/src/vidxp/cli_commands/__init__.py b/src/vidxp/cli_commands/__init__.py new file mode 100644 index 0000000..48c7dec --- /dev/null +++ b/src/vidxp/cli_commands/__init__.py @@ -0,0 +1 @@ +"""Typer command groups for the VidXP CLI adapter.""" diff --git a/src/vidxp/cli_commands/index.py b/src/vidxp/cli_commands/index.py new file mode 100644 index 0000000..f552757 --- /dev/null +++ b/src/vidxp/cli_commands/index.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Iterable + +import typer + +from vidxp.cli_support import ( + CLIState, + IndexProgress, + OutputFormat, + effective_output_format, + emit_json, + emit_status, + parse_capability_options, + selected_modalities, + state_from_context, +) + + +app = typer.Typer(no_args_is_help=True, help="Manage a local video index.") + + +def create_index( + state: CLIState, + path: Path, + *, + modalities: Iterable[str], + frame_stride: int, + capability_options: dict[str, dict], +) -> dict: + show_progress = ( + not state.quiet and state.output_format == OutputFormat.rich + ) + with IndexProgress(show_progress) as progress: + summary = state.service.create_index( + path, + modalities=modalities, + frame_stride=frame_stride, + capability_options=capability_options, + progress_callback=progress.update, + ) + if state.output_format == OutputFormat.json: + emit_json(summary) + else: + typer.secho( + "Video indexing completed successfully.", + fg=typer.colors.GREEN, + bold=True, + ) + return summary + + +@app.command("create") +def index_create( + ctx: typer.Context, + path: Annotated[ + Path, + typer.Argument( + exists=True, + dir_okay=False, + readable=True, + resolve_path=True, + help="Local video file to index.", + ), + ], + modalities: Annotated[ + list[str] | None, + typer.Option( + "--modality", + "-m", + help="Modality to index; repeat to select more than one.", + ), + ] = None, + frame_stride: Annotated[ + int, + typer.Option( + "--frame-stride", + min=1, + help="Materialize every Nth frame for visual modalities.", + ), + ] = 1, + capability_options: Annotated[ + list[str] | None, + typer.Option( + "--option", + help=( + "Capability setting as CAPABILITY.KEY=VALUE; " + "repeat for multiple settings." + ), + ), + ] = None, +) -> None: + """Create or replace a local index for one video.""" + + state = state_from_context(ctx) + create_index( + state, + path, + modalities=selected_modalities(modalities), + frame_stride=frame_stride, + capability_options=parse_capability_options(capability_options), + ) + + +@app.command("status") +def index_status( + ctx: typer.Context, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Show the state and source of the selected index.""" + + state = state_from_context(ctx) + emit_status( + state.service.index_status(), + output_format=effective_output_format(state, json_output), + ) + + +@app.command("clear") +def index_clear( + ctx: typer.Context, + yes: Annotated[ + bool, + typer.Option("--yes", "-y", help="Skip the confirmation prompt."), + ] = False, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Clear indexed records and VidXP run state.""" + + state = state_from_context(ctx) + if not yes: + typer.confirm( + f"Clear the local index at {state.service.index_directory}?", + abort=True, + ) + cleared = state.service.clear_index() + payload = { + "cleared": cleared, + "index_directory": str(state.service.index_directory), + } + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(payload) + else: + typer.echo("Index cleared." if cleared else "No index was found.") diff --git a/src/vidxp/cli_commands/repositories.py b/src/vidxp/cli_commands/repositories.py new file mode 100644 index 0000000..7e89a66 --- /dev/null +++ b/src/vidxp/cli_commands/repositories.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Annotated + +import typer +from rich.console import Console +from rich.table import Table + +from vidxp.cli_support import ( + OutputFormat, + effective_output_format, + emit_json, + state_from_context, +) + + +app = typer.Typer( + no_args_is_help=True, + help="Manage named index repositories.", +) + + +def _emit_repository( + repository, + *, + output_format: OutputFormat, +) -> None: + payload = repository.to_dict() + if output_format == OutputFormat.json: + emit_json(payload) + return + table = Table(title=f"Repository {repository.name}") + table.add_column("Field") + table.add_column("Value") + table.add_row("Index directory", str(repository.index_directory)) + table.add_row("Device", repository.device or "default") + table.add_row("Configured", "yes" if repository.configured else "no") + Console().print(table) + + +@app.command("list") +def repositories_list( + ctx: typer.Context, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """List configured repositories and the active selection.""" + + state = state_from_context(ctx) + repositories = state.registry.list() + payload = { + "active_repository": state.repository.name, + "config_file": str(state.registry.path), + "repositories": [item.to_dict() for item in repositories], + } + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(payload) + return + table = Table(title="VidXP repositories") + table.add_column("Active") + table.add_column("Name") + table.add_column("Index directory") + table.add_column("Device") + for repository in repositories: + table.add_row( + "*" if repository.name == state.repository.name else "", + repository.name, + str(repository.index_directory), + repository.device or "default", + ) + Console().print(table) + + +@app.command("show") +def repositories_show( + ctx: typer.Context, + name: Annotated[ + str | None, + typer.Argument(help="Repository name; defaults to the active one."), + ] = None, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Show one repository configuration.""" + + state = state_from_context(ctx) + repository = ( + state.registry.resolve(name) + if name is not None + else state.repository + ) + _emit_repository( + repository, + output_format=effective_output_format(state, json_output), + ) + + +@app.command("add") +def repositories_add( + ctx: typer.Context, + name: Annotated[str, typer.Argument(help="Repository name.")], + index_directory: Annotated[ + Path, + typer.Option( + "--index-dir", + file_okay=False, + help="Local index directory managed by this repository.", + ), + ], + device: Annotated[ + str | None, + typer.Option("--device", help="Optional repository device."), + ] = None, + replace: Annotated[ + bool, + typer.Option("--replace", help="Replace an existing configuration."), + ] = False, + use: Annotated[ + bool, + typer.Option("--use", help="Make this repository active."), + ] = False, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Add or replace a named local index repository.""" + + state = state_from_context(ctx) + repository = state.registry.add( + name, + index_directory, + device=device, + replace=replace, + ) + if use: + repository = state.registry.use(repository.name) + _emit_repository( + repository, + output_format=effective_output_format(state, json_output), + ) + + +@app.command("use") +def repositories_use( + ctx: typer.Context, + name: Annotated[str, typer.Argument(help="Repository name.")], + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Select the default repository for future commands.""" + + state = state_from_context(ctx) + repository = state.registry.use(name) + _emit_repository( + repository, + output_format=effective_output_format(state, json_output), + ) + + +@app.command("remove") +def repositories_remove( + ctx: typer.Context, + name: Annotated[str, typer.Argument(help="Repository name.")], + yes: Annotated[ + bool, + typer.Option("--yes", "-y", help="Skip the confirmation prompt."), + ] = False, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Remove configuration without deleting indexed data.""" + + state = state_from_context(ctx) + repository = state.registry.resolve(name) + if not repository.configured: + raise typer.BadParameter( + "The implicit default repository has no saved configuration." + ) + if not yes: + typer.confirm( + f"Remove repository configuration {repository.name!r}? " + "Indexed data will not be deleted.", + abort=True, + ) + removed = state.registry.remove(repository.name) + payload = { + "removed": removed.to_dict(), + "index_deleted": False, + } + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(payload) + else: + typer.echo( + f"Removed {removed.name!r}; indexed data was left untouched." + ) diff --git a/src/vidxp/cli_commands/runtime.py b/src/vidxp/cli_commands/runtime.py new file mode 100644 index 0000000..94a97e6 --- /dev/null +++ b/src/vidxp/cli_commands/runtime.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import os +from typing import Annotated + +import typer + +from vidxp.capabilities.registry import CAPABILITIES, capability_names +from vidxp.cli_support import ( + OutputFormat, + effective_output_format, + emit_json, + parse_capability_options, + parse_modalities, + state_from_context, +) + +ALL_CAPABILITIES = ",".join(capability_names()) +PREPARABLE_CAPABILITIES = ",".join( + name + for name, capability in CAPABILITIES.items() + if capability.prepare is not None +) + + +def doctor( + ctx: typer.Context, + modalities: Annotated[ + str, + typer.Option( + "--modalities", + "-m", + help="Only validate dependencies for these modalities.", + ), + ] = ALL_CAPABILITIES, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Validate selected indexing dependencies without downloading models.""" + + selected = parse_modalities(modalities) + state = state_from_context(ctx) + result = state.service.check_dependencies(selected) + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(result) + else: + for check in result["checks"]: + if check["ok"]: + detail = f": {check['path']}" if check.get("path") else "" + typer.secho( + f"OK {check['name']}{detail}", + fg=typer.colors.GREEN, + ) + else: + typer.secho( + f"FAILED {check['name']}: {check['error']}", + fg=typer.colors.RED, + ) + if not result["ok"]: + raise typer.Exit(1) + if effective_output_format(state, json_output) == OutputFormat.rich: + typer.secho( + "Selected VidXP dependencies are available.", + fg=typer.colors.GREEN, + bold=True, + ) + + +def prepare( + ctx: typer.Context, + modalities: Annotated[ + str, + typer.Option( + "--modalities", + "-m", + help="Only prepare models for these modalities.", + ), + ] = PREPARABLE_CAPABILITIES, + capability_options: Annotated[ + list[str] | None, + typer.Option( + "--option", + help=( + "Capability setting as CAPABILITY.KEY=VALUE; " + "repeat for multiple settings." + ), + ), + ] = None, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, +) -> None: + """Download and cache selected runtime models before indexing.""" + + selected = parse_modalities(modalities) + state = state_from_context(ctx) + result = state.service.prepare_models( + selected, + capability_options=parse_capability_options(capability_options), + progress_callback=( + None + if state.quiet + or effective_output_format(state, json_output) + == OutputFormat.json + else lambda event: typer.echo(event["message"]) + ), + ) + if effective_output_format(state, json_output) == OutputFormat.json: + emit_json(result) + else: + typer.secho( + "Selected VidXP runtime models are prepared.", + fg=typer.colors.GREEN, + bold=True, + ) + + +def ui( + ctx: typer.Context, + host: Annotated[ + str | None, + typer.Option("--host", help="Streamlit server address."), + ] = None, + port: Annotated[ + int | None, + typer.Option( + "--port", + min=1, + max=65535, + help="Streamlit server port.", + ), + ] = None, +) -> None: + """Launch Streamlit with the selected repository configuration.""" + + state = state_from_context(ctx) + os.environ["VIDXP_CONFIG_FILE"] = str(state.registry.path) + os.environ["VIDXP_REPOSITORY"] = state.repository.name + os.environ["VIDXP_INDEX_DIR"] = str(state.service.index_directory) + if state.service.device is None: + os.environ.pop("VIDXP_DEVICE", None) + else: + os.environ["VIDXP_DEVICE"] = state.service.device + + try: + from vidxp import frontend + except ModuleNotFoundError as exc: + if exc.name == "streamlit": + raise RuntimeError( + "The browser interface requires the frontend extra. " + "Install vidxp[frontend]." + ) from exc + raise + + frontend.SERVICE = state.service + frontend.SAVED_VIDEO_PATH = ( + state.service.index_directory / "source-video.mp4" + ) + frontend.ACTOR_OUTPUT_PATH = ( + state.service.index_directory / "actor-result.mp4" + ) + streamlit_arguments = [] + if host is not None: + streamlit_arguments.append(f"--server.address={host}") + if port is not None: + streamlit_arguments.append(f"--server.port={port}") + frontend.main(streamlit_arguments) diff --git a/src/vidxp/cli_commands/search.py b/src/vidxp/cli_commands/search.py new file mode 100644 index 0000000..d2c9f90 --- /dev/null +++ b/src/vidxp/cli_commands/search.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from typing import Annotated, Callable + +import typer + +from vidxp.capabilities.registry import CAPABILITIES +from vidxp.capabilities.schemas import SearchResult +from vidxp.cli_support import ( + CLIState, + effective_output_format, + emit_search, + state_from_context, +) + + +app = typer.Typer(no_args_is_help=True, help="Search the active index.") + + +def run_search( + state: CLIState, + capability: str, + query: str, + *, + top_k: int, + json_output: bool, +) -> SearchResult: + result = state.service.search(capability, query, top_k=top_k) + emit_search( + result, + output_format=effective_output_format(state, json_output), + ) + return result + + +def _search_command(capability: str) -> Callable: + def command( + ctx: typer.Context, + query: Annotated[ + str, + typer.Argument(help="Text query to find."), + ], + top_k: Annotated[ + int, + typer.Option( + "--top-k", + "-k", + min=1, + help="Maximum ranked hits.", + ), + ] = 10, + json_output: Annotated[ + bool, + typer.Option("--json", help="Emit machine-readable JSON."), + ] = False, + ) -> None: + run_search( + state_from_context(ctx), + capability, + query, + top_k=top_k, + json_output=json_output, + ) + + command.__name__ = f"search_{capability}" + command.__doc__ = CAPABILITIES[capability].description + return command + + +for _name, _capability in CAPABILITIES.items(): + if "search" in _capability.operations: + app.command(_name)(_search_command(_name)) diff --git a/src/vidxp/cli_support.py b/src/vidxp/cli_support.py new file mode 100644 index 0000000..67947f9 --- /dev/null +++ b/src/vidxp/cli_support.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from enum import Enum +from typing import Any, Iterable + +import typer +from rich.console import Console +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, +) +from rich.table import Table + +from vidxp.application import VidXPService +from vidxp.capabilities.registry import ( + index_capability_names, + validate_capability_names, +) +from vidxp.capabilities.schemas import SearchResult +from vidxp.repositories import RepositoryConfig, RepositoryRegistry + + +class OutputFormat(str, Enum): + rich = "rich" + json = "json" + + +@dataclass +class CLIState: + service: VidXPService + registry: RepositoryRegistry + repository: RepositoryConfig + output_format: OutputFormat = OutputFormat.rich + quiet: bool = False + + +def state_from_context(ctx: typer.Context) -> CLIState: + state = ctx.ensure_object(CLIState) + if not isinstance(state, CLIState): + raise RuntimeError("VidXP CLI state was not initialized.") + return state + + +def effective_output_format( + state: CLIState, + json_output: bool, +) -> OutputFormat: + return OutputFormat.json if json_output else state.output_format + + +def emit_json(payload: Any) -> None: + typer.echo( + json.dumps( + payload, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + + +def emit_search( + result: SearchResult, + *, + output_format: OutputFormat, +) -> None: + if output_format == OutputFormat.json: + emit_json(result.to_dict()) + return + if not result.hits: + typer.echo(f"No {result.modality} matches found.") + return + table = Table(title=f"{result.modality.title()} search results") + table.add_column("Rank", justify="right") + table.add_column("Start", justify="right") + table.add_column("End", justify="right") + table.add_column("Score", justify="right") + table.add_column("Video") + for hit in result.hits: + table.add_row( + str(hit.rank), + f"{hit.start:.3f}s", + f"{hit.end:.3f}s", + f"{hit.score:.6f}", + hit.video_id, + ) + Console().print(table) + + +def emit_status( + status: dict[str, Any], + *, + output_format: OutputFormat, +) -> None: + if output_format == OutputFormat.json: + emit_json(status) + return + table = Table(title="VidXP index") + table.add_column("Field") + table.add_column("Value") + table.add_row("State", str(status.get("state", "unknown"))) + table.add_row("Message", str(status.get("message", "—"))) + if updated_at := status.get("updated_at"): + table.add_row("Updated", str(updated_at)) + if video := status.get("video"): + table.add_row( + "Video", + str(video.get("source_name") or video.get("path") or "—"), + ) + summary = status.get("summary") or {} + if modalities := (summary.get("configuration") or {}).get( + "enabled_modalities" + ): + table.add_row("Modalities", ", ".join(map(str, modalities))) + Console().print(table) + + +class IndexProgress: + def __init__(self, enabled: bool) -> None: + self.enabled = enabled + self.progress = Progress( + SpinnerColumn(), + TextColumn("{task.description}"), + BarColumn(), + TaskProgressColumn(), + console=Console(stderr=True), + ) + self.task_id: int | None = None + self.stage: str | None = None + + def __enter__(self) -> "IndexProgress": + if self.enabled: + self.progress.start() + return self + + def __exit__(self, *_: Any) -> None: + if self.enabled: + self.progress.stop() + + def update(self, event: dict[str, Any]) -> None: + if not self.enabled: + return + stage = str(event.get("stage", "indexing")) + total = event.get("total") + current = event.get("current") + if self.task_id is None or stage != self.stage: + if self.task_id is not None: + self.progress.update(self.task_id, completed=1, total=1) + self.task_id = self.progress.add_task( + str(event.get("message", stage.replace("_", " "))), + total=float(total) if total else None, + ) + self.stage = stage + self.progress.update( + self.task_id, + description=str(event.get("message", stage.replace("_", " "))), + total=float(total) if total else None, + completed=float(current) if current is not None else None, + ) + + +def selected_modalities( + values: Iterable[str] | None, +) -> tuple[str, ...]: + if values is None: + return index_capability_names() + try: + return validate_capability_names(values) + except ValueError as exc: + raise typer.BadParameter(str(exc)) from exc + + +def parse_modalities(value: str) -> tuple[str, ...]: + selected = tuple( + item.strip().lower() + for item in value.split(",") + if item.strip() + ) + try: + return validate_capability_names(selected) + except ValueError as exc: + raise typer.BadParameter(str(exc)) from exc + + +def parse_capability_options( + values: Iterable[str] | None, +) -> dict[str, dict[str, Any]]: + options: dict[str, dict[str, Any]] = {} + for value in values or (): + path, separator, raw = value.partition("=") + capability, dot, key = path.partition(".") + if not separator or not dot or not capability or not key: + raise typer.BadParameter( + "Capability options must use CAPABILITY.KEY=VALUE." + ) + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = raw + options.setdefault(capability, {})[key] = parsed + return options diff --git a/src/vidxp/core/__init__.py b/src/vidxp/core/__init__.py index 192cb11..37bb927 100644 --- a/src/vidxp/core/__init__.py +++ b/src/vidxp/core/__init__.py @@ -1,17 +1,13 @@ -"""Benchmark-ready indexing and retrieval engine.""" +"""Shared runtime primitives used by VidXP capabilities.""" from vidxp.core.contracts import ( CancellationToken, IndexConfig, - SearchHit, - SearchResult, VideoSource, ) __all__ = [ "CancellationToken", "IndexConfig", - "SearchHit", - "SearchResult", "VideoSource", ] diff --git a/src/vidxp/core/actor_results.py b/src/vidxp/core/actor_results.py deleted file mode 100644 index 5bc7b27..0000000 --- a/src/vidxp/core/actor_results.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from vidxp.core.contracts import IndexConfig -from vidxp.core.storage import IndexStorage -from vidxp.core.video import render_actor_video - - -class ActorClusterNotFoundError(LookupError): - """Raised when an actor cluster has no retained detections.""" - - -@dataclass(frozen=True) -class ActorRenderResult: - output_path: Path - detection_count: int - - -def actor_detections( - config: IndexConfig, - cluster_id: str, - *, - storage: IndexStorage | None = None, -) -> list[dict]: - if config.video_id is None: - raise ValueError("IndexConfig.video_id is required for actor results.") - owns_storage = storage is None - active_storage = storage or IndexStorage(config) - try: - records = active_storage.actor_detections( - video_id=config.video_id, - cluster_id=cluster_id, - ) - finally: - if owns_storage: - active_storage.close() - - return [ - { - **record, - "bbox": ( - int(record["bbox_top"]), - int(record["bbox_right"]), - int(record["bbox_bottom"]), - int(record["bbox_left"]), - ), - } - for record in records - ] - - -def render_actor_result( - config: IndexConfig, - cluster_id: str, - input_path: str | Path, - output_path: str | Path, - *, - storage: IndexStorage | None = None, -) -> ActorRenderResult: - detections = actor_detections(config, cluster_id, storage=storage) - if not detections: - raise ActorClusterNotFoundError( - f"Actor cluster {cluster_id} was not found in the completed index." - ) - destination = Path(output_path) - render_actor_video(input_path, destination, cluster_id, detections) - return ActorRenderResult(destination, len(detections)) diff --git a/src/vidxp/core/contracts.py b/src/vidxp/core/contracts.py index fe8fdfa..cece2ce 100644 --- a/src/vidxp/core/contracts.py +++ b/src/vidxp/core/contracts.py @@ -11,9 +11,8 @@ from urllib.parse import quote -INDEX_SCHEMA_VERSION = 2 +INDEX_SCHEMA_VERSION = 3 MANIFEST_SCHEMA_VERSION = 1 -SUPPORTED_MODALITIES = ("dialogue", "scene", "actor") class IndexCancelledError(RuntimeError): @@ -73,29 +72,18 @@ class IndexConfig: split: str = "local" run_id: str = "default" video_id: str | None = None - enabled_modalities: tuple[str, ...] = SUPPORTED_MODALITIES + enabled_modalities: tuple[str, ...] = () frame_stride: int = 1 - dialogue_words_per_phrase: int = 5 - scene_batch_size: int = 32 - dialogue_batch_size: int = 128 - transcription_batch_size: int = 16 - actor_batch_size: int = 16 storage_batch_size: int = 256 - normalize_dialogue_embeddings: bool = True vector_distance: str = "l2" - face_match_threshold: float = 0.55 - face_num_jitters: int = 2 - actor_min_detections: int = 4 device: str = "cpu" - sentence_model: str = "sentence-transformers/all-MiniLM-L6-v2" - whisper_model: str = "large-v2" - clip_model: str = "ViT-B/32" + capability_options: Mapping[str, Mapping[str, Any]] = field( + default_factory=dict + ) output_root: str | Path = "benchmark_runs" storage_directory: str | Path | None = None - collection_names: tuple[str, str, str] = ( - "dialogue", - "scene", - "actor", + collection_names: Mapping[str, str] = field( + default_factory=dict ) def __post_init__(self) -> None: @@ -112,40 +100,61 @@ def __post_init__(self) -> None: _require_identifier("video_id", self.video_id) modalities = tuple(dict.fromkeys(self.enabled_modalities)) - unknown = sorted(set(modalities) - set(SUPPORTED_MODALITIES)) - if unknown: - raise ValueError(f"Unsupported indexing modalities: {', '.join(unknown)}") if not modalities: raise ValueError("At least one indexing modality must be enabled.") object.__setattr__(self, "enabled_modalities", modalities) + collection_names = { + str(capability): str(name) + for capability, name in self.collection_names.items() + } + if not collection_names: + collection_names = { + capability: capability + for capability in modalities + } + missing_collections = sorted( + set(modalities) - set(collection_names) + ) + if missing_collections: + raise ValueError( + "Missing collection names for capabilities: " + + ", ".join(missing_collections) + ) + object.__setattr__(self, "collection_names", collection_names) + capability_options = { + str(capability): dict(options) + for capability, options in self.capability_options.items() + } + unknown_options = sorted( + set(capability_options) - set(modalities) + ) + if unknown_options: + raise ValueError( + "Options were supplied for disabled capabilities: " + + ", ".join(unknown_options) + ) + object.__setattr__( + self, + "capability_options", + capability_options, + ) for label in ( "frame_stride", - "dialogue_words_per_phrase", - "scene_batch_size", - "dialogue_batch_size", - "transcription_batch_size", - "actor_batch_size", "storage_batch_size", - "face_num_jitters", - "actor_min_detections", ): if getattr(self, label) <= 0: raise ValueError(f"{label} must be greater than zero.") - if not 0 < self.face_match_threshold < 1: - raise ValueError("face_match_threshold must be between zero and one.") if self.vector_distance not in {"l2", "cosine", "ip"}: raise ValueError( "vector_distance must be one of: l2, cosine, ip." ) - if len(self.collection_names) != len(SUPPORTED_MODALITIES): - raise ValueError("collection_names must define dialogue, scene, and actor.") collection_pattern = re.compile( r"^[A-Za-z0-9][A-Za-z0-9._-]{1,510}[A-Za-z0-9]$" ) invalid_names = [ name - for name in self.collection_names + for name in self.collection_names.values() if not collection_pattern.fullmatch(str(name)) ] if invalid_names: @@ -154,7 +163,9 @@ def __post_init__(self) -> None: "with an alphanumeric character, and contain only " "letters, numbers, periods, underscores, or hyphens." ) - if len(set(self.collection_names)) != len(self.collection_names): + if len(set(self.collection_names.values())) != len( + self.collection_names + ): raise ValueError("collection_names must be distinct.") @classmethod @@ -163,12 +174,11 @@ def local(cls, **changes: Any) -> "IndexConfig": defaults = { "storage_directory": "chroma_data", - "collection_names": ( - "voiceEmbeddings", - "sceneEmbeddings", - "actorCollection", - ), } + if "enabled_modalities" not in changes: + from vidxp.capabilities.registry import index_capability_names + + defaults["enabled_modalities"] = index_capability_names() defaults.update(changes) return cls(**defaults) @@ -198,8 +208,10 @@ def record_identity( ) -> dict[str, str]: if self.video_id is None: raise ValueError("IndexConfig.video_id is required for record metadata.") - if modality not in SUPPORTED_MODALITIES: - raise ValueError(f"Unsupported indexing modality: {modality}") + if modality not in self.enabled_modalities: + raise ValueError( + f"Capability {modality!r} is not enabled for this run." + ) return { "dataset": self.dataset, "split": self.split, @@ -209,10 +221,17 @@ def record_identity( "source_id": source_id, } + def options_for(self, capability: str) -> dict[str, Any]: + if capability not in self.enabled_modalities: + raise ValueError( + f"Capability {capability!r} is not enabled for this run." + ) + return dict(self.capability_options.get(capability, {})) + def to_dict(self) -> dict[str, Any]: payload = asdict(self) payload["enabled_modalities"] = list(self.enabled_modalities) - payload["collection_names"] = list(self.collection_names) + payload["collection_names"] = dict(self.collection_names) payload["run_directory"] = str(self.run_directory) payload["index_directory"] = str(self.index_directory) return payload @@ -262,52 +281,6 @@ class StorageRecord: document: str | None = None -@dataclass(frozen=True) -class SearchHit: - rank: int - video_id: str - start: float - end: float - score: float - raw_distance: float - modality: str - source_id: str - metadata: Mapping[str, Any] = field(default_factory=dict) - - def to_dict(self) -> dict[str, Any]: - return { - "rank": self.rank, - "video_id": self.video_id, - "start": self.start, - "end": self.end, - "score": self.score, - "raw_distance": self.raw_distance, - "modality": self.modality, - "source_id": self.source_id, - "metadata": dict(self.metadata), - } - - -@dataclass(frozen=True) -class SearchResult: - query_id: str - query: str - modality: str - hits: tuple[SearchHit, ...] - - def to_dict(self) -> dict[str, Any]: - return { - "schema_version": INDEX_SCHEMA_VERSION, - "query_id": self.query_id, - "query": self.query, - "modality": self.modality, - "hits": [hit.to_dict() for hit in self.hits], - } - - def to_prediction(self) -> dict[str, list[dict[str, Any]]]: - return {self.query_id: [hit.to_dict() for hit in self.hits]} - - class CancellationToken: """A small cooperative cancellation token checked between work batches.""" diff --git a/src/vidxp/core/indexing.py b/src/vidxp/core/indexing.py deleted file mode 100644 index 5ecd836..0000000 --- a/src/vidxp/core/indexing.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Public indexing entry points. - -Implementation details live in the modality-specific modules. These wrappers -remain for callers that index one visual modality at a time. -""" - -from __future__ import annotations - -from typing import Any - -from vidxp.core.contracts import VideoSource -from vidxp.core.indexing_dialogue import ( - DialoguePhrase, - build_dialogue_phrases, - index_dialogue, - transcribe_video, -) -from vidxp.core.indexing_visual import VisualIndexResult, index_visuals - - -def index_scenes(source: VideoSource, **options: Any) -> dict[str, Any]: - return dict(index_visuals(source, modalities=("scene",), **options).summary) - - -def index_actors(source: VideoSource, **options: Any) -> dict[str, Any]: - return dict(index_visuals(source, modalities=("actor",), **options).summary) - - -__all__ = [ - "DialoguePhrase", - "VisualIndexResult", - "build_dialogue_phrases", - "index_actors", - "index_dialogue", - "index_scenes", - "index_visuals", - "transcribe_video", -] diff --git a/src/vidxp/core/indexing_visual.py b/src/vidxp/core/indexing_visual.py deleted file mode 100644 index 6238151..0000000 --- a/src/vidxp/core/indexing_visual.py +++ /dev/null @@ -1,324 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from time import perf_counter -from typing import Any, Mapping, Sequence - -from vidxp.core.contracts import ( - CancellationToken, - IndexConfig, - StorageRecord, - VideoSource, - batched, - stable_source_id, -) -from vidxp.core.indexing_actor import ( - ActorIndexState, - finalize_actor_index, - process_actor_samples, -) -from vidxp.core.indexing_common import ProgressCallback, report_progress -from vidxp.core.models import get_clip_model -from vidxp.core.storage import IndexStorage -from vidxp.core.video import ( - FrameSample, - FrameStreamStats, - iter_frame_batches, - probe_video, -) - - -@dataclass -class SceneIndexState: - model: Any - preprocess: Any - stored_frames: int = 0 - - -@dataclass(frozen=True) -class VisualIndexResult: - summary: Mapping[str, Any] - timings: Mapping[str, float] - - -def _encode_scene_batch(samples, model, preprocess, device): - import torch - from PIL import Image - - images = torch.stack( - [preprocess(Image.fromarray(sample.frame)) for sample in samples] - ).to(device) - with torch.no_grad(): - features = model.encode_image(images) - features /= features.norm(dim=-1, keepdim=True) - return features.cpu().numpy().tolist() - - -def _scene_records( - samples, - vectors, - info, - config: IndexConfig, -) -> list[StorageRecord]: - records = [] - for sample, vector in zip(samples, vectors): - end = min( - info.duration, - sample.timestamp + config.frame_stride / info.fps, - ) - if end <= sample.timestamp: - end = sample.timestamp + 1 / info.fps - source_id = stable_source_id( - config.run_id, - str(config.video_id), - "scene", - f"f{sample.frame_index:012d}", - ) - records.append( - StorageRecord( - source_id=source_id, - embedding=vector, - metadata={ - **config.record_identity("scene", source_id), - "frame_index": sample.frame_index, - "timestamp": sample.timestamp, - "start": sample.timestamp, - "end": end, - "fps": info.fps, - "duration": info.duration, - }, - ) - ) - return records - - -def _rgb_samples(samples) -> list[FrameSample]: - import cv2 - - return [ - FrameSample( - frame_index=sample.frame_index, - timestamp=sample.timestamp, - frame=cv2.cvtColor(sample.frame, cv2.COLOR_BGR2RGB), - ) - for sample in samples - ] - - -def _process_scene_samples( - samples, - *, - state: SceneIndexState, - info, - config: IndexConfig, - storage: IndexStorage, - cancellation: CancellationToken, -) -> None: - for group in batched(samples, config.scene_batch_size): - cancellation.raise_if_cancelled() - vectors = _encode_scene_batch( - group, - state.model, - state.preprocess, - config.device, - ) - state.stored_frames += storage.upsert( - "scene", - _scene_records(group, vectors, info, config), - batch_size=config.storage_batch_size, - cancellation=cancellation, - ) - - -def _consume_visual_stream( - source: VideoSource, - *, - selected: tuple[str, ...], - expected: int, - info, - scene_state: SceneIndexState | None, - actor_state: ActorIndexState | None, - config: IndexConfig, - storage: IndexStorage, - cancellation: CancellationToken, - progress: ProgressCallback | None, - timings: dict[str, float], -) -> FrameStreamStats: - stream_stats = FrameStreamStats() - decode_batch_size = max( - config.scene_batch_size if "scene" in selected else 0, - config.actor_batch_size if "actor" in selected else 0, - ) - stream = iter( - iter_frame_batches( - source.path, - frame_stride=config.frame_stride, - batch_size=decode_batch_size, - cancellation=cancellation, - stats=stream_stats, - ) - ) - while True: - stream_started = perf_counter() - try: - samples = next(stream) - except StopIteration: - timings["frame_stream"] += perf_counter() - stream_started - break - rgb_samples = _rgb_samples(samples) - timings["frame_stream"] += perf_counter() - stream_started - - if scene_state is not None: - scene_started = perf_counter() - _process_scene_samples( - rgb_samples, - state=scene_state, - info=info, - config=config, - storage=storage, - cancellation=cancellation, - ) - timings["scene"] += perf_counter() - scene_started - - if actor_state is not None: - actor_started = perf_counter() - process_actor_samples( - rgb_samples, - state=actor_state, - config=config, - storage=storage, - cancellation=cancellation, - ) - timings["actor"] += perf_counter() - actor_started - - report_progress( - progress, - "visual_indexing", - "Indexing the shared sampled-frame stream.", - stream_stats.frames_materialized, - expected, - ) - return stream_stats - - -def _visual_summary( - *, - scene_state: SceneIndexState | None, - actor_state: ActorIndexState | None, - stream_stats: FrameStreamStats, - actor_detections: int, - actor_clusters: int, - info, -) -> dict[str, Any]: - scene_frames = scene_state.stored_frames if scene_state is not None else 0 - actor_frames = ( - actor_state.processed_frames if actor_state is not None else 0 - ) - sampled_frames = ( - stream_stats.frames_materialized - or max(scene_frames, actor_frames) - ) - return { - "source_frames_advanced": stream_stats.frames_advanced, - "sampled_frames": sampled_frames, - "processed_frames": sampled_frames, - "frame_operations": scene_frames + actor_frames, - "scene_frames": scene_frames, - "actor_frames": actor_frames, - "actor_detections": actor_detections, - "actor_clusters": actor_clusters, - "duration": info.duration, - "fps": info.fps, - } - - -def index_visuals( - source: VideoSource, - *, - config: IndexConfig, - storage: IndexStorage, - cancellation: CancellationToken, - progress: ProgressCallback | None = None, - modalities: Sequence[str] | None = None, -) -> VisualIndexResult: - if config.video_id is None: - raise ValueError("IndexConfig.video_id is required for indexing.") - if source.path is None: - raise ValueError("Scene and actor indexing require a video path.") - - selected = tuple( - modality - for modality in ( - config.enabled_modalities if modalities is None else modalities - ) - if modality in {"scene", "actor"} - ) - if not selected: - raise ValueError("At least one visual modality must be selected.") - - started = perf_counter() - info = probe_video(source.path) - expected = (info.frame_count + config.frame_stride - 1) // config.frame_stride - scene_state = None - actor_state = ActorIndexState() if "actor" in selected else None - timings = {"frame_stream": 0.0, "scene": 0.0, "actor": 0.0} - - if "scene" in selected: - scene_started = perf_counter() - report_progress( - progress, - "preparing_scene_model", - f"Preparing scene model: CLIP {config.clip_model}.", - ) - scene_model, scene_preprocess = get_clip_model( - config.clip_model, - config.device, - ) - scene_state = SceneIndexState(scene_model, scene_preprocess) - timings["scene"] += perf_counter() - scene_started - - report_progress( - progress, - "visual_indexing", - "Decoding sampled frames for " - + " and ".join(selected) - + " indexing.", - 0, - expected, - ) - stream_stats = _consume_visual_stream( - source, - selected=selected, - expected=expected, - info=info, - scene_state=scene_state, - actor_state=actor_state, - config=config, - storage=storage, - cancellation=cancellation, - progress=progress, - timings=timings, - ) - - actor_detections = actor_clusters = 0 - if actor_state is not None: - actor_started = perf_counter() - actor_detections, actor_clusters = finalize_actor_index( - actor_state, - config=config, - storage=storage, - ) - timings["actor"] += perf_counter() - actor_started - timings["visual_total"] = perf_counter() - started - - return VisualIndexResult( - summary=_visual_summary( - scene_state=scene_state, - actor_state=actor_state, - stream_stats=stream_stats, - actor_detections=actor_detections, - actor_clusters=actor_clusters, - info=info, - ), - timings=timings, - ) diff --git a/src/vidxp/core/manifest.py b/src/vidxp/core/manifest.py index 3afccd8..d2a35f0 100644 --- a/src/vidxp/core/manifest.py +++ b/src/vidxp/core/manifest.py @@ -11,13 +11,16 @@ from pathlib import Path from typing import Any, Mapping +from vidxp.capabilities.registry import ( + get_capability, + runtime_distributions, +) from vidxp.core.contracts import ( INDEX_SCHEMA_VERSION, MANIFEST_SCHEMA_VERSION, IndexConfig, VideoSource, ) -from vidxp.core.models import runtime_distributions MANIFEST_FILE = "manifest.json" @@ -212,19 +215,11 @@ def _model_manifest( ], ) -> dict[str, Any]: models: dict[str, Any] = {"device": self.config.device} - if "dialogue" in self.config.enabled_modalities: - models["dialogue"] = self.config.sentence_model - if any(source.transcript is None for _, source, _, _ in sources): - models["transcription"] = self.config.whisper_model - if "scene" in self.config.enabled_modalities: - models["scene"] = self.config.clip_model - if "actor" in self.config.enabled_modalities: - models["actor"] = { - "library": "face_recognition", - "match_threshold": self.config.face_match_threshold, - "num_jitters": self.config.face_num_jitters, - "minimum_detections": self.config.actor_min_detections, - } + source_values = tuple(source for _, source, _, _ in sources) + for name in self.config.enabled_modalities: + manifest = get_capability(name).model_manifest + if manifest is not None: + models.update(manifest(self.config, source_values)) return models def initialize( diff --git a/src/vidxp/core/models.py b/src/vidxp/core/models.py deleted file mode 100644 index b275f6e..0000000 --- a/src/vidxp/core/models.py +++ /dev/null @@ -1,150 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from functools import lru_cache -from importlib import import_module - - -@dataclass(frozen=True) -class RuntimeDependency: - label: str - module: str - distribution: str - - -INDEXING_DEPENDENCIES = { - "dialogue": ( - RuntimeDependency("ChromaDB", "chromadb", "chromadb"), - RuntimeDependency( - "Sentence Transformers", - "sentence_transformers", - "sentence-transformers", - ), - ), - "scene": ( - RuntimeDependency("ChromaDB", "chromadb", "chromadb"), - RuntimeDependency("CLIP", "clip", "clip-anytorch"), - RuntimeDependency("NumPy", "numpy", "numpy"), - RuntimeDependency("OpenCV", "cv2", "opencv-python"), - RuntimeDependency("Pillow", "PIL.Image", "Pillow"), - RuntimeDependency("PyTorch", "torch", "torch"), - ), - "actor": ( - RuntimeDependency("ChromaDB", "chromadb", "chromadb"), - RuntimeDependency( - "face recognition", - "face_recognition", - "face-recognition", - ), - RuntimeDependency("NumPy", "numpy", "numpy"), - RuntimeDependency("OpenCV", "cv2", "opencv-python"), - ), - "transcription": ( - RuntimeDependency("MoviePy", "moviepy.editor", "moviepy"), - RuntimeDependency("WhisperX", "whisperx", "whisperx"), - ), -} - -PROVENANCE_ONLY_DISTRIBUTIONS = ( - "dlib", - "face-recognition-models", - "filelock", -) - - -@lru_cache -def get_embedder(model_name: str, device: str): - from sentence_transformers import SentenceTransformer - - return SentenceTransformer(model_name, device=device) - - -@lru_cache -def get_clip_model(model_name: str, device: str): - import clip - - return clip.load(model_name, device=device) - - -@lru_cache -def get_whisper_model(model_name: str, device: str): - import whisperx - - return whisperx.load_model(model_name, device, compute_type="float32") - - -@lru_cache -def get_alignment_model(language: str, device: str): - import whisperx - - return whisperx.load_align_model(language_code=language, device=device) - - -def selected_dependencies( - modalities: tuple[str, ...], - *, - needs_transcription: bool, -) -> tuple[RuntimeDependency, ...]: - dependencies = [ - dependency - for modality in modalities - for dependency in INDEXING_DEPENDENCIES[modality] - ] - if needs_transcription: - dependencies.extend(INDEXING_DEPENDENCIES["transcription"]) - return tuple( - { - dependency.module: dependency - for dependency in dependencies - }.values() - ) - - -def runtime_distributions() -> tuple[str, ...]: - distributions = { - dependency.distribution - for group in INDEXING_DEPENDENCIES.values() - for dependency in group - } - distributions.update(PROVENANCE_ONLY_DISTRIBUTIONS) - return tuple(sorted(distributions, key=str.lower)) - - -def clear_model_cache() -> None: - get_embedder.cache_clear() - get_clip_model.cache_clear() - get_whisper_model.cache_clear() - get_alignment_model.cache_clear() - - -def dependency_failures( - modalities: tuple[str, ...], - *, - needs_transcription: bool, -) -> list[tuple[str, str]]: - failures = [] - for dependency in selected_dependencies( - modalities, - needs_transcription=needs_transcription, - ): - try: - import_module(dependency.module) - except Exception as exc: - failures.append( - (dependency.label, f"{type(exc).__name__}: {exc}") - ) - return failures - - -def require_dependencies( - modalities: tuple[str, ...], - *, - needs_transcription: bool, -) -> None: - failures = dependency_failures( - modalities, - needs_transcription=needs_transcription, - ) - if failures: - details = "; ".join(f"{label}: {error}" for label, error in failures) - raise RuntimeError(f"Indexing dependencies are unavailable: {details}") diff --git a/src/vidxp/core/runner.py b/src/vidxp/core/runner.py index 21f393c..896652a 100644 --- a/src/vidxp/core/runner.py +++ b/src/vidxp/core/runner.py @@ -7,6 +7,10 @@ from filelock import FileLock, Timeout +from vidxp.capabilities.registry import ( + get_capability, + require_dependencies, +) from vidxp.core.contracts import ( INDEX_SCHEMA_VERSION, CancellationToken, @@ -15,15 +19,12 @@ IndexSchemaError, VideoSource, ) -from vidxp.core.indexing_dialogue import index_dialogue -from vidxp.core.indexing_visual import index_visuals from vidxp.core.manifest import ( ManifestStore, combined_checksum, source_checksum, source_checksums, ) -from vidxp.core.models import require_dependencies from vidxp.core.storage import IndexStorage from vidxp.index_state import ( IndexingInProgressError, @@ -74,7 +75,11 @@ def indexing_in_progress(config: IndexConfig | None = None) -> bool: return _run_lock_held(active_config.run_directory) -def local_config_from_status(status: dict[str, Any]) -> IndexConfig: +def local_config_from_status( + status: dict[str, Any], + *, + storage_directory: str | Path | None = None, +) -> IndexConfig: summary = status.get("summary") or {} if summary.get("index_schema_version") != INDEX_SCHEMA_VERSION: raise IndexSchemaError( @@ -93,13 +98,16 @@ def local_config_from_status(status: dict[str, Any]) -> IndexConfig: "split": str(summary["split"]), "run_id": str(summary["run_id"]), "video_id": str(summary["video_id"]), - "storage_directory": "chroma_data", } ) + if storage_directory is not None: + stored["storage_directory"] = str(storage_directory) + else: + stored.setdefault("storage_directory", "chroma_data") if "enabled_modalities" in stored: stored["enabled_modalities"] = tuple(stored["enabled_modalities"]) if "collection_names" in stored: - stored["collection_names"] = tuple(stored["collection_names"]) + stored["collection_names"] = dict(stored["collection_names"]) return IndexConfig(**stored) @@ -128,8 +136,8 @@ def _report( callback(event) -def _run_modality( - modality: str, +def _run_capability_group( + names: tuple[str, ...], source: VideoSource, config: IndexConfig, storage: IndexStorage, @@ -137,6 +145,21 @@ def _run_modality( cancellation: CancellationToken, progress_callback: ProgressCallback | None, ) -> dict[str, Any]: + definitions = tuple(get_capability(name) for name in names) + indexer = definitions[0].indexer + index_stage = definitions[0].index_stage + if indexer is None or index_stage is None: + raise ValueError( + f"Capability {names[0]!r} does not support indexing." + ) + if any(definition.indexer is not indexer for definition in definitions): + raise RuntimeError("Grouped capabilities must share one indexer.") + if any( + definition.index_stage != index_stage + for definition in definitions + ): + raise RuntimeError("Grouped capabilities must share one index stage.") + started = perf_counter() active_substage: str | None = None substage_started = started @@ -162,94 +185,83 @@ def stage_progress(event: dict[str, Any]) -> None: ) try: - if modality != "dialogue": - raise ValueError(f"Unsupported non-visual modality: {modality}") - stats = index_dialogue( + result = indexer( source, config=config, storage=storage, cancellation=cancellation, progress=stage_progress, + modalities=names, ) except BaseException: - if active_substage is not None: + if active_substage is not None and active_substage != index_stage: manifest.record_stage( str(config.video_id), active_substage, perf_counter() - substage_started, {"state": "incomplete"}, ) + manifest.record_stage( + str(config.video_id), + index_stage, + perf_counter() - started, + {"state": "incomplete", "capabilities": list(names)}, + ) raise - if active_substage is not None: + if active_substage is not None and active_substage != index_stage: manifest.record_stage( str(config.video_id), active_substage, perf_counter() - substage_started, {}, ) - manifest.record_stage( - str(config.video_id), - modality, - perf_counter() - started, - stats, - ) - return stats - - -def _run_visual_modalities( - modalities: tuple[str, ...], - source: VideoSource, - config: IndexConfig, - storage: IndexStorage, - manifest: ManifestStore, - cancellation: CancellationToken, - progress_callback: ProgressCallback | None, -) -> dict[str, Any]: - started = perf_counter() - - def report(event: dict[str, Any]) -> None: - _report( - progress_callback, - {**event, "video_id": config.video_id}, - ) - - try: - result = index_visuals( - source, - config=config, - storage=storage, - cancellation=cancellation, - progress=report, - modalities=modalities, - ) - except BaseException: + for stage_name, duration in result.timings.items(): + if stage_name.endswith("_total"): + continue manifest.record_stage( str(config.video_id), - "visual_indexing", - perf_counter() - started, - {"state": "incomplete", "modalities": list(modalities)}, + stage_name, + float(duration), + {}, ) - raise - - stats = dict(result.summary) - timings = dict(result.timings) - for stage_name in ("frame_stream", "scene", "actor"): - if stage_name in timings and ( - stage_name == "frame_stream" or stage_name in modalities - ): - manifest.record_stage( - str(config.video_id), - stage_name, - float(timings[stage_name]), - {}, - ) manifest.record_stage( str(config.video_id), - "visual_indexing", - float(timings.get("visual_total", perf_counter() - started)), - stats, + index_stage, + float( + result.timings.get( + f"{index_stage.removesuffix('_indexing')}_total", + result.timings.get( + "visual_total", + perf_counter() - started, + ), + ) + ), + result.summary, ) - return stats + return dict(result.summary) + + +def _index_groups(names: tuple[str, ...]) -> tuple[tuple[str, ...], ...]: + groups: list[list[str]] = [] + handlers = [] + for name in names: + handler = get_capability(name).indexer + if handler is None: + raise ValueError( + f"Capability {name!r} does not support indexing." + ) + try: + group_index = next( + index + for index, existing in enumerate(handlers) + if existing is handler + ) + except StopIteration: + handlers.append(handler) + groups.append([name]) + else: + groups[group_index].append(name) + return tuple(tuple(group) for group in groups) def _run_enabled_modalities( @@ -262,36 +274,12 @@ def _run_enabled_modalities( set_stage: Callable[[str], None], ) -> dict[str, Any]: summary: dict[str, Any] = {} - visual_modalities = tuple( - modality - for modality in config.enabled_modalities - if modality in {"scene", "actor"} - ) - visual_complete = False - for modality in config.enabled_modalities: + for names in _index_groups(config.enabled_modalities): cancellation.raise_if_cancelled() - if modality in visual_modalities: - if visual_complete: - continue - set_stage("visual_indexing") - summary.update( - _run_visual_modalities( - visual_modalities, - source, - config, - storage, - manifest, - cancellation, - progress_callback, - ) - ) - visual_complete = True - continue - - set_stage(f"{modality}_indexing") + set_stage(get_capability(names[0]).index_stage) summary.update( - _run_modality( - modality, + _run_capability_group( + names, source, config, storage, @@ -303,19 +291,6 @@ def _run_enabled_modalities( return summary -def _normalize_frame_summary(summary: dict[str, Any]) -> None: - scene_frames = int(summary.get("scene_frames", 0)) - actor_frames = int(summary.get("actor_frames", 0)) - summary.setdefault("sampled_frames", max(scene_frames, actor_frames)) - summary.setdefault("processed_frames", int(summary["sampled_frames"])) - summary.setdefault("frame_operations", scene_frames + actor_frames) - summary.setdefault( - "source_frames_advanced", - int(summary.get("decoded_frames", 0)) - + int(summary.get("actor_decoded_frames", 0)), - ) - - def _process_video( video_id: str, source: VideoSource, @@ -344,10 +319,7 @@ def set_stage(value: str) -> None: cancellation.raise_if_cancelled() require_dependencies( config.enabled_modalities, - needs_transcription=( - "dialogue" in config.enabled_modalities - and source.transcript is None - ), + source=source, ) stage = "preparing_storage" for modality in config.enabled_modalities: @@ -364,7 +336,6 @@ def set_stage(value: str) -> None: set_stage, ) ) - _normalize_frame_summary(summary) manifest.complete_video( video_id, checksum=checksum, @@ -542,6 +513,7 @@ def report(event: dict[str, Any]) -> None: total=event.get("total"), summary=event.get("summary"), error=event.get("error"), + index_directory=active_config.index_directory, ) if progress_callback is not None: progress_callback(event) diff --git a/src/vidxp/core/storage.py b/src/vidxp/core/storage.py index 3abc12d..50ac3db 100644 --- a/src/vidxp/core/storage.py +++ b/src/vidxp/core/storage.py @@ -48,7 +48,7 @@ def __init__(self, config: IndexConfig): self.path = config.index_directory self.path.mkdir(parents=True, exist_ok=True) self.client = _client_for_path(str(self.path.resolve())) - self._names = dict(zip(("dialogue", "scene", "actor"), config.collection_names)) + self._names = dict(config.collection_names) self._collections: dict[str, Any] = {} def close(self) -> None: @@ -104,12 +104,18 @@ def delete_video(self, modality: str, video_id: str) -> None: where=metadata_filter(self.config, video_id=video_id), ) - def delete_actor_cluster(self, video_id: str, cluster_id: str) -> None: - self.collection("actor").delete( + def delete_records( + self, + modality: str, + *, + video_id: str, + filters: Mapping[str, Any] | None = None, + ) -> None: + self.collection(modality).delete( where=metadata_filter( self.config, video_id=video_id, - extra={"cluster_id": cluster_id}, + extra=filters, ), ) @@ -179,29 +185,26 @@ def query( for source_id, metadata, distance in zip(ids, metadatas, distances) ] - def actor_detections( + def records( self, + modality: str, *, - video_id: str, - cluster_id: str, + video_id: str | None = None, + filters: Mapping[str, Any] | None = None, ) -> list[dict[str, Any]]: - result = self.collection("actor").get( + result = self.collection(modality).get( where=metadata_filter( self.config, video_id=video_id, - extra={"cluster_id": cluster_id}, + extra=filters, ), include=["metadatas"], ) - records = [ + return [ dict(metadata) for metadata in (result.get("metadatas") or []) if metadata ] - return sorted( - records, - key=lambda item: (int(item["frame_index"]), item["detection_id"]), - ) def size_bytes(self) -> int: return directory_size(self.path) diff --git a/src/vidxp/dependencies.py b/src/vidxp/dependencies.py new file mode 100644 index 0000000..cdc8826 --- /dev/null +++ b/src/vidxp/dependencies.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from functools import lru_cache +from importlib.metadata import ( + PackageNotFoundError, + requires as distribution_requirements, + version, +) +from importlib.resources import files +from typing import Iterable + +from packaging.requirements import Requirement + + +@lru_cache(maxsize=None) +def packaged_requirements( + package: str, + resource: str = "requirements.txt", +) -> tuple[Requirement, ...]: + content = files(package).joinpath(resource).read_text( + encoding="utf-8" + ) + return tuple( + Requirement(line) + for raw_line in content.splitlines() + if (line := raw_line.strip()) and not line.startswith("#") + ) + + +def active_requirements( + requirements: Iterable[Requirement], + *, + extra: str = "", +) -> tuple[Requirement, ...]: + environment = {"extra": extra} + return tuple( + requirement + for requirement in requirements + if requirement.marker is None + or requirement.marker.evaluate(environment) + ) + + +def inspect_requirement(requirement: Requirement) -> dict: + try: + installed = version(requirement.name) + except PackageNotFoundError: + return { + "name": requirement.name, + "requirement": str(requirement), + "installed_version": None, + "ok": False, + "error": "distribution is not installed", + } + if requirement.specifier and not requirement.specifier.contains( + installed, + prereleases=True, + ): + return { + "name": requirement.name, + "requirement": str(requirement), + "installed_version": installed, + "ok": False, + "error": ( + f"installed version {installed} does not satisfy " + f"{requirement.specifier}" + ), + } + return { + "name": requirement.name, + "requirement": str(requirement), + "installed_version": installed, + "ok": True, + "error": None, + } + + +def requirements_available(package: str) -> bool: + return all( + inspect_requirement(requirement)["ok"] + for requirement in active_requirements( + packaged_requirements(package) + ) + ) + + +def installed_base_requirements( + distribution: str = "vidxp", +) -> tuple[Requirement, ...]: + try: + declared = distribution_requirements(distribution) or () + except PackageNotFoundError: + return () + return active_requirements( + (Requirement(value) for value in declared), + ) diff --git a/src/vidxp/frontend.py b/src/vidxp/frontend.py index a52a622..84a26e2 100644 --- a/src/vidxp/frontend.py +++ b/src/vidxp/frontend.py @@ -1,24 +1,32 @@ import hashlib import sys from pathlib import Path +from typing import Sequence import streamlit as st -from vidxp.core.actor_results import render_actor_result -from vidxp.core.runner import local_config_from_status -from vidxp.core.search import search_dialogue, search_scene -from vidxp.index_state import ( - IndexNotReadyError, - read_index_status, -) +from vidxp.application import VidXPService +from vidxp.capabilities.registry import index_capability_names +from vidxp.index_state import IndexNotReadyError from vidxp.index_worker import ( cancel_indexing, indexing_in_progress, start_indexing, ) +from vidxp.repositories import resolve_repository + + +def _configured_service() -> VidXPService: + _, repository = resolve_repository() + return VidXPService( + repository.index_directory, + device=repository.device, + ) -SAVED_VIDEO_PATH = Path("video.mp4") -ACTOR_OUTPUT_PATH = Path("output.mp4") + +SERVICE = _configured_service() +SAVED_VIDEO_PATH = SERVICE.index_directory / "source-video.mp4" +ACTOR_OUTPUT_PATH = SERVICE.index_directory / "actor-result.mp4" INDEX_REQUESTED_KEY = "_vidxp_index_requested" INDEX_ERROR_KEY = "_vidxp_index_error" SEARCH_RESULT_KEY = "_vidxp_search_result" @@ -76,7 +84,7 @@ def _render_index_status(status, active, uploaded_video, request_error=None): "message": "Indexing is running.", } _render_progress(event) - elif not status: + elif not status or status.get("state") == "missing": st.caption("First indexing may download missing runtime model weights.") elif status["state"] == "ready": if _is_search_ready(status, uploaded_video): @@ -115,9 +123,18 @@ def _request_cancellation(): ) -def _run_indexing(uploaded_video, status): +def _available_index_modalities() -> tuple[str, ...]: + return tuple( + name + for name in index_capability_names() + if SERVICE.check_dependencies((name,))["ok"] + ) + + +def _run_indexing(uploaded_video, status, modalities): try: if uploaded_video is not None: + SAVED_VIDEO_PATH.parent.mkdir(parents=True, exist_ok=True) SAVED_VIDEO_PATH.write_bytes(uploaded_video.getvalue()) source_name = uploaded_video.name else: @@ -126,7 +143,12 @@ def _run_indexing(uploaded_video, status): if status else SAVED_VIDEO_PATH.name ) - start_indexing(str(SAVED_VIDEO_PATH), source_name) + start_indexing( + str(SAVED_VIDEO_PATH), + source_name, + SERVICE, + modalities=modalities, + ) except Exception as exc: st.session_state[INDEX_ERROR_KEY] = f"{type(exc).__name__}: {exc}" else: @@ -138,14 +160,12 @@ def _run_indexing(uploaded_video, status): def _run_search(search_type, query): try: - status = read_index_status() - if not status or status.get("state") != "ready": + status = SERVICE.index_status() + if status.get("state") != "ready": raise IndexNotReadyError("The video index is not ready.") - config = local_config_from_status(status) if search_type == "actor": ACTOR_OUTPUT_PATH.unlink(missing_ok=True) - render_actor_result( - config, + SERVICE.render_actor( query, SAVED_VIDEO_PATH, ACTOR_OUTPUT_PATH, @@ -161,12 +181,10 @@ def _run_search(search_type, query): "video_path": str(ACTOR_OUTPUT_PATH), } - finder = search_dialogue if search_type == "dialogue" else search_scene - result = finder( + result = SERVICE.search( + search_type, query, - config=config, top_k=1, - video_id=config.video_id, ) if not result.hits: return {"error": f"No {search_type} match was found."} @@ -275,13 +293,15 @@ def run(): st.set_page_config(page_title="VidXP", page_icon="🎬", layout="wide") st.title("VidXP") st.caption("Index and search video by dialogue, scene, and actor.") + st.caption(f"Index repository: {SERVICE.index_directory}") - active = indexing_in_progress() + active = indexing_in_progress(SERVICE) if not active: st.session_state.pop(CANCEL_REQUESTED_KEY, None) requested = st.session_state.get(INDEX_REQUESTED_KEY, False) busy = active or requested - status = read_index_status() + status = SERVICE.index_status() + installed_modalities = _available_index_modalities() video_column, workflow_column = st.columns( [0.95, 1.05], gap="large", @@ -293,10 +313,25 @@ def run(): with workflow_column: st.subheader("Build index") + selected_modalities = tuple( + st.multiselect( + "Capabilities", + installed_modalities, + default=installed_modalities, + disabled=busy, + help="Install another capability extra to make it available here.", + ) + ) + if not installed_modalities: + st.warning( + "No indexing capabilities are installed. " + 'Install one, for example: pip install "vidxp[scene]"' + ) st.button( "Index video", type="primary", disabled=busy + or not selected_modalities or (uploaded_video is None and not SAVED_VIDEO_PATH.is_file()), help=( "Indexing is already running." @@ -323,9 +358,9 @@ def run(): @st.fragment(run_every="1s") def poll_index_status(): - latest_active = indexing_in_progress() + latest_active = indexing_in_progress(SERVICE) _render_index_status( - read_index_status(), + SERVICE.index_status(), latest_active, uploaded_video, st.session_state.get(INDEX_ERROR_KEY), @@ -366,13 +401,18 @@ def poll_index_status(): _render_search_result(st.session_state.get(SEARCH_RESULT_KEY)) if requested: - _run_indexing(uploaded_video, status) + _run_indexing(uploaded_video, status, selected_modalities) -def main(): +def main(arguments: Sequence[str] = ()): from streamlit.web import cli as streamlit_cli - sys.argv = ["streamlit", "run", str(Path(__file__).resolve()), *sys.argv[1:]] + sys.argv = [ + "streamlit", + "run", + str(Path(__file__).resolve()), + *arguments, + ] raise SystemExit(streamlit_cli.main()) diff --git a/src/vidxp/index_worker.py b/src/vidxp/index_worker.py index a4c3b0d..d96a9f4 100644 --- a/src/vidxp/index_worker.py +++ b/src/vidxp/index_worker.py @@ -2,42 +2,73 @@ from multiprocessing.process import BaseProcess from threading import Lock +from vidxp.application import VidXPService from vidxp.core.contracts import CancellationToken -from vidxp.core.runner import ( - index_video, - indexing_in_progress as in_process_indexing, -) from vidxp.index_state import IndexingInProgressError +from vidxp.repositories import resolve_repository _process: BaseProcess | None = None _cancel_event = None _start_lock = Lock() -def _run_indexing(path: str, source_name: str, cancel_event) -> None: - index_video( +def _configured_service() -> VidXPService: + _, repository = resolve_repository() + return VidXPService( + repository.index_directory, + device=repository.device, + ) + + +def _run_indexing( + path: str, + source_name: str, + cancel_event, + index_directory: str, + device: str | None, + modalities: tuple[str, ...], +) -> None: + VidXPService(index_directory, device=device).create_index( path, + modalities=modalities, source_name=source_name, cancellation=CancellationToken(cancel_event), ) -def indexing_in_progress() -> bool: - return (_process is not None and _process.is_alive()) or in_process_indexing() +def indexing_in_progress(service: VidXPService | None = None) -> bool: + active_service = service or _configured_service() + return ( + _process is not None and _process.is_alive() + ) or active_service.indexing_in_progress() -def start_indexing(path: str, source_name: str) -> None: +def start_indexing( + path: str, + source_name: str, + service: VidXPService | None = None, + *, + modalities: tuple[str, ...], +) -> None: global _cancel_event, _process + active_service = service or _configured_service() with _start_lock: - if indexing_in_progress(): + if indexing_in_progress(active_service): raise IndexingInProgressError("Another video is already being indexed.") context = get_context("spawn") _cancel_event = context.Event() _process = context.Process( target=_run_indexing, - args=(path, source_name, _cancel_event), + args=( + path, + source_name, + _cancel_event, + str(active_service.index_directory), + active_service.device, + modalities, + ), name="vidxp-indexer", daemon=True, ) diff --git a/src/vidxp/repositories.py b/src/vidxp/repositories.py new file mode 100644 index 0000000..b0e4df7 --- /dev/null +++ b/src/vidxp/repositories.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import json +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from filelock import FileLock + + +REPOSITORY_SCHEMA_VERSION = 1 +DEFAULT_REPOSITORY_NAME = "default" +DEFAULT_INDEX_DIRECTORY = Path("chroma_data") +_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") + + +class RepositoryConfigError(ValueError): + """Raised when repository configuration is invalid or unavailable.""" + + +def default_config_path() -> Path: + configured = os.environ.get("VIDXP_CONFIG_FILE") + if configured: + return Path(configured).expanduser() + if sys.platform == "win32": + root = Path(os.environ.get("APPDATA", Path.home() / "AppData/Roaming")) + elif sys.platform == "darwin": + root = Path.home() / "Library/Application Support" + else: + root = Path( + os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config") + ) + return root / "vidxp" / "repositories.json" + + +def _repository_name(value: str) -> str: + name = str(value).strip() + if not _NAME_PATTERN.fullmatch(name): + raise RepositoryConfigError( + "Repository names must be 1-64 characters, start with a letter " + "or number, and contain only letters, numbers, '.', '_', or '-'." + ) + return name + + +@dataclass(frozen=True) +class RepositoryConfig: + name: str + index_directory: Path + device: str | None = None + configured: bool = True + + def __post_init__(self) -> None: + object.__setattr__(self, "name", _repository_name(self.name)) + object.__setattr__( + self, + "index_directory", + Path(self.index_directory).expanduser(), + ) + if self.device is not None and not str(self.device).strip(): + raise RepositoryConfigError("Repository device must not be empty.") + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "type": "local", + "index_directory": str(self.index_directory), + "device": self.device, + "configured": self.configured, + } + + +class RepositoryRegistry: + def __init__(self, path: str | Path | None = None) -> None: + self.path = Path(path) if path is not None else default_config_path() + + def read(self) -> dict[str, Any]: + return self._read_unlocked() + + def _read_unlocked(self) -> dict[str, Any]: + if not self.path.is_file(): + return { + "schema_version": REPOSITORY_SCHEMA_VERSION, + "active_repository": None, + "repositories": {}, + } + try: + payload = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RepositoryConfigError( + f"Repository configuration is unreadable: {self.path}" + ) from exc + if ( + not isinstance(payload, dict) + or payload.get("schema_version") != REPOSITORY_SCHEMA_VERSION + or not isinstance(payload.get("repositories"), dict) + ): + raise RepositoryConfigError( + f"Unsupported repository configuration: {self.path}" + ) + self._validate_payload(payload) + return payload + + def list(self) -> tuple[RepositoryConfig, ...]: + payload = self.read() + configured = tuple( + self._from_entry(name, entry) + for name, entry in sorted(payload["repositories"].items()) + ) + if any(item.name == DEFAULT_REPOSITORY_NAME for item in configured): + return configured + return ( + RepositoryConfig( + DEFAULT_REPOSITORY_NAME, + DEFAULT_INDEX_DIRECTORY, + configured=False, + ), + *configured, + ) + + def resolve(self, name: str | None = None) -> RepositoryConfig: + payload = self.read() + selected = ( + _repository_name(name) + if name is not None + else payload.get("active_repository") or DEFAULT_REPOSITORY_NAME + ) + entry = payload["repositories"].get(selected) + if entry is not None: + return self._from_entry(selected, entry) + if selected == DEFAULT_REPOSITORY_NAME: + return RepositoryConfig( + DEFAULT_REPOSITORY_NAME, + DEFAULT_INDEX_DIRECTORY, + configured=False, + ) + raise RepositoryConfigError( + f"Repository {selected!r} is not configured." + ) + + def add( + self, + name: str, + index_directory: str | Path, + *, + device: str | None = None, + replace: bool = False, + ) -> RepositoryConfig: + repository = RepositoryConfig( + name, + Path(index_directory).expanduser().resolve(), + device, + ) + with self._lock(): + payload = self._read_unlocked() + if repository.name in payload["repositories"] and not replace: + raise RepositoryConfigError( + f"Repository {repository.name!r} already exists." + ) + payload["repositories"][repository.name] = { + "type": "local", + "index_directory": str(repository.index_directory), + "device": repository.device, + } + self._write_unlocked(payload) + return repository + + def remove(self, name: str) -> RepositoryConfig: + selected = _repository_name(name) + with self._lock(): + payload = self._read_unlocked() + entry = payload["repositories"].pop(selected, None) + if entry is None: + raise RepositoryConfigError( + f"Repository {selected!r} is not configured." + ) + if payload.get("active_repository") == selected: + payload["active_repository"] = None + self._write_unlocked(payload) + return self._from_entry(selected, entry) + + def use(self, name: str) -> RepositoryConfig: + selected = _repository_name(name) + with self._lock(): + payload = self._read_unlocked() + entry = payload["repositories"].get(selected) + if entry is not None: + repository = self._from_entry(selected, entry) + payload["active_repository"] = repository.name + elif selected == DEFAULT_REPOSITORY_NAME: + repository = RepositoryConfig( + DEFAULT_REPOSITORY_NAME, + DEFAULT_INDEX_DIRECTORY, + configured=False, + ) + payload["active_repository"] = None + else: + raise RepositoryConfigError( + f"Repository {selected!r} is not configured." + ) + self._write_unlocked(payload) + return repository + + def write(self, payload: Mapping[str, Any]) -> None: + with self._lock(): + self._write_unlocked(payload) + + def _write_unlocked(self, payload: Mapping[str, Any]) -> None: + validated = dict(payload) + self._validate_payload(validated) + self.path.parent.mkdir(parents=True, exist_ok=True) + temporary = self.path.with_suffix(self.path.suffix + ".tmp") + temporary.write_text( + json.dumps( + validated, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + temporary.replace(self.path) + + def _lock(self) -> FileLock: + self.path.parent.mkdir(parents=True, exist_ok=True) + return FileLock(str(self.path) + ".lock") + + @staticmethod + def _from_entry( + name: str, + entry: Mapping[str, Any], + ) -> RepositoryConfig: + return RepositoryConfig( + name=name, + index_directory=Path(str(entry["index_directory"])), + device=( + str(entry["device"]) + if entry.get("device") is not None + else None + ), + ) + + @staticmethod + def _validate_payload(payload: Mapping[str, Any]) -> None: + repositories = payload.get("repositories") + if not isinstance(repositories, dict): + raise RepositoryConfigError( + "Repository configuration must contain a repositories object." + ) + for name, entry in repositories.items(): + _repository_name(name) + if ( + not isinstance(entry, dict) + or entry.get("type") != "local" + or not str(entry.get("index_directory", "")).strip() + ): + raise RepositoryConfigError( + f"Repository {name!r} has an invalid local configuration." + ) + active = payload.get("active_repository") + if active is not None: + active = _repository_name(str(active)) + if active not in repositories: + raise RepositoryConfigError( + f"Active repository {active!r} is not configured." + ) + + +def resolve_repository( + *, + registry_path: str | Path | None = None, + name: str | None = None, + index_directory: str | Path | None = None, + device: str | None = None, +) -> tuple[RepositoryRegistry, RepositoryConfig]: + registry = RepositoryRegistry(registry_path) + selected_name = name or os.environ.get("VIDXP_REPOSITORY") + repository = registry.resolve(selected_name) + resolved_index = ( + Path(index_directory) + if index_directory is not None + else Path(os.environ["VIDXP_INDEX_DIR"]) + if os.environ.get("VIDXP_INDEX_DIR") + else repository.index_directory + ) + resolved_device = ( + device + if device is not None + else os.environ.get("VIDXP_DEVICE") or repository.device + ) + return registry, RepositoryConfig( + name=repository.name, + index_directory=resolved_index, + device=resolved_device, + configured=repository.configured, + ) diff --git a/src/vidxp/requirements/frontend.txt b/src/vidxp/requirements/frontend.txt new file mode 100644 index 0000000..c8038a5 --- /dev/null +++ b/src/vidxp/requirements/frontend.txt @@ -0,0 +1 @@ +streamlit>=1.37 diff --git a/src/vidxp/requirements/storage.txt b/src/vidxp/requirements/storage.txt new file mode 100644 index 0000000..99812b1 --- /dev/null +++ b/src/vidxp/requirements/storage.txt @@ -0,0 +1 @@ +chromadb diff --git a/tests/test_actor_results.py b/tests/test_actor_results.py index 0bde93b..d916368 100644 --- a/tests/test_actor_results.py +++ b/tests/test_actor_results.py @@ -3,8 +3,9 @@ from pathlib import Path from unittest.mock import Mock, patch -from vidxp.core.actor_results import ( +from vidxp.capabilities.actor.results import ( ActorClusterNotFoundError, + actor_clusters, actor_detections, render_actor_result, ) @@ -17,13 +18,22 @@ def setUp(self): def test_actor_detection_metadata_is_converted_once(self): storage = Mock() - storage.actor_detections.return_value = [ + storage.records.return_value = [ { + "detection_id": "d2", + "cluster_id": "3", "frame_index": 2, + "timestamp": 0.2, "bbox_top": 1, "bbox_right": 4, "bbox_bottom": 5, "bbox_left": 0, + "dataset": "local", + "split": "local", + "run_id": "default", + "video_id": "video-1", + "modality": "actor", + "source_id": "actor:d2", } ] @@ -33,16 +43,35 @@ def test_actor_detection_metadata_is_converted_once(self): storage=storage, ) - self.assertEqual(detections[0]["bbox"], (1, 4, 5, 0)) - storage.actor_detections.assert_called_once_with( + self.assertEqual(detections[0].bbox, (1, 4, 5, 0)) + storage.records.assert_called_once_with( + "actor", video_id="video-1", - cluster_id="3", + filters={"cluster_id": "3"}, ) storage.close.assert_not_called() + def test_actor_clusters_summarize_detection_ranges(self): + storage = Mock() + storage.records.return_value = [ + {"cluster_id": "1", "timestamp": 4.5}, + {"cluster_id": "1", "timestamp": 1.5}, + {"cluster_id": "2", "timestamp": 9.0}, + ] + + clusters = actor_clusters(self.config, storage=storage) + + self.assertEqual( + [cluster.cluster_id for cluster in clusters], + ["1", "2"], + ) + self.assertEqual(clusters[0].detection_count, 2) + self.assertEqual(clusters[0].first_timestamp, 1.5) + self.assertEqual(clusters[0].last_timestamp, 4.5) + def test_render_actor_result_rejects_an_empty_cluster(self): storage = Mock() - storage.actor_detections.return_value = [] + storage.records.return_value = [] with self.assertRaises(ActorClusterNotFoundError): render_actor_result( @@ -55,19 +84,28 @@ def test_render_actor_result_rejects_an_empty_cluster(self): def test_render_actor_result_returns_output_details(self): storage = Mock() - storage.actor_detections.return_value = [ + storage.records.return_value = [ { + "detection_id": "d2", + "cluster_id": "3", "frame_index": 2, + "timestamp": 0.2, "bbox_top": 1, "bbox_right": 4, "bbox_bottom": 5, "bbox_left": 0, + "dataset": "local", + "split": "local", + "run_id": "default", + "video_id": "video-1", + "modality": "actor", + "source_id": "actor:d2", } ] with TemporaryDirectory() as directory: output = Path(directory) / "actor.mp4" with patch( - "vidxp.core.actor_results.render_actor_video" + "vidxp.capabilities.actor.results.render_actor_video" ) as renderer: result = render_actor_result( self.config, diff --git a/tests/test_application.py b/tests/test_application.py new file mode 100644 index 0000000..5ea2650 --- /dev/null +++ b/tests/test_application.py @@ -0,0 +1,300 @@ +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from vidxp.application import VidXPService +from vidxp.capabilities.contracts import ( + CapabilityDefinition, + OperationDefinition, + PreparationContext, +) +from vidxp.capabilities.scene.config import SceneConfig +from vidxp.capabilities.schemas import SearchInput, SearchResult +from vidxp.core.contracts import IndexConfig + + +class ApplicationServiceTests(unittest.TestCase): + def test_execute_supports_validated_operation_without_an_index(self): + capability = CapabilityDefinition( + name="export", + description="Export results.", + extra="export", + operations={ + "run": OperationDefinition( + input_model=SearchInput, + output_model=SearchResult, + handler=lambda context, request: { + "query_id": "export:1", + "query": request.query, + "modality": "export", + "hits": (), + }, + requires_index=False, + ) + }, + ) + service = VidXPService() + with ( + patch( + "vidxp.application.get_capability", + return_value=capability, + ), + patch.object( + service, + "active_config", + side_effect=AssertionError("index should not be loaded"), + ), + ): + result = service.execute( + "export", + "run", + {"query": "result bundle"}, + ) + + self.assertEqual(result.query, "result bundle") + + def test_missing_index_has_a_stable_status_contract(self): + service = VidXPService("missing-index") + with patch( + "vidxp.application.read_index_status", + return_value=None, + ): + status = service.index_status() + + self.assertEqual(status["state"], "missing") + self.assertEqual(status["schema_version"], 1) + + def test_active_config_uses_selected_directory_and_device(self): + service = VidXPService("selected-index", device="cuda") + ready = {"state": "ready"} + stored_config = IndexConfig.local( + video_id="video-1", + storage_directory="selected-index", + ) + with ( + patch( + "vidxp.application.require_ready_index", + return_value=ready, + ) as require, + patch( + "vidxp.application.local_config_from_status", + return_value=stored_config, + ) as restore, + ): + config, status = service.active_config() + + require.assert_called_once_with(Path("selected-index")) + restore.assert_called_once_with( + ready, + storage_directory=Path("selected-index"), + ) + self.assertEqual(config.device, "cuda") + self.assertIs(status, ready) + + def test_search_is_a_thin_adapter_over_the_core(self): + service = VidXPService() + expected = SearchResult( + query_id="scene:1", + query="yellow taxi", + modality="scene", + ) + with patch.object( + service, + "execute", + return_value=expected, + ) as execute: + result = service.search("scene", "yellow taxi", top_k=7) + + self.assertIs(result, expected) + execute.assert_called_once_with( + "scene", + "search", + {"query": "yellow taxi", "top_k": 7}, + ) + + def test_execute_translates_missing_optional_dependency(self): + def missing_dependency(_context, _request): + raise ModuleNotFoundError("No module named 'clip'", name="clip") + + operation = OperationDefinition( + input_model=SearchInput, + output_model=SearchResult, + handler=missing_dependency, + requires_index=False, + ) + capability = CapabilityDefinition( + name="scene", + description="Search visual scenes.", + extra="scene", + operations={"search": operation}, + ) + service = VidXPService() + with patch( + "vidxp.application.get_capability", + return_value=capability, + ): + with self.assertRaisesRegex( + RuntimeError, + r'clip is unavailable.*pip install "vidxp\[scene\]"', + ): + service.execute( + "scene", + "search", + {"query": "yellow taxi"}, + ) + + def test_create_index_centralizes_storage_and_runtime_configuration(self): + service = VidXPService("selected-index", device="cuda") + with patch( + "vidxp.application.index_video", + return_value={"scene_frames": 1}, + ) as index: + summary = service.create_index( + "video.mp4", + modalities=("scene",), + frame_stride=5, + ) + + self.assertEqual(summary, {"scene_frames": 1}) + config = index.call_args.kwargs["config"] + self.assertEqual(config.storage_directory, "selected-index") + self.assertEqual(config.device, "cuda") + self.assertEqual(config.enabled_modalities, ("scene",)) + self.assertEqual(config.frame_stride, 5) + + def test_dependency_checks_return_a_transport_neutral_contract(self): + service = VidXPService() + with patch( + "vidxp.application.dependency_checks", + return_value=( + {"name": "CLIP", "ok": False, "error": "missing"}, + ), + ): + result = service.check_dependencies(("scene",)) + + self.assertFalse(result["ok"]) + failed = [check for check in result["checks"] if not check["ok"]] + self.assertEqual(failed, [ + {"name": "CLIP", "ok": False, "error": "missing"} + ]) + + def test_model_preparation_reports_progress_without_cli_dependencies(self): + service = VidXPService(device="cuda") + events = [] + prepare = Mock( + side_effect=lambda context, progress: ( + progress( + { + "state": "preparing", + "stage": "scene_model", + "message": "Preparing scene model", + } + ), + (SceneConfig.model_validate(context.settings).model,), + )[1] + ) + capability = Mock( + prepare=prepare, + config_model=SceneConfig, + ) + with ( + patch( + "vidxp.application.dependency_checks", + return_value=(), + ), + patch( + "vidxp.application.get_capability", + return_value=capability, + ), + ): + result = service.prepare_models( + ("scene",), + progress_callback=events.append, + ) + + prepare.assert_called_once() + context = prepare.call_args.args[0] + self.assertIsInstance(context, PreparationContext) + self.assertEqual(context.device, "cuda") + self.assertEqual(result["device"], "cuda") + self.assertEqual(events[0]["stage"], "scene_model") + + def test_clear_removes_only_known_run_state_after_clearing_collections(self): + with TemporaryDirectory() as directory: + index_directory = Path(directory) / "index" + index_directory.mkdir() + for name in ( + "index_status.json", + "manifest.json", + "timings.jsonl", + "failures.jsonl", + "run.complete.json", + ): + (index_directory / name).write_text("{}", encoding="utf-8") + unrelated = index_directory / "keep.txt" + unrelated.write_text("keep", encoding="utf-8") + checkpoint_directory = index_directory / "checkpoints" + checkpoint_directory.mkdir() + (checkpoint_directory / "one.json").write_text( + "{}", + encoding="utf-8", + ) + + storage = Mock() + storage.__enter__ = Mock(return_value=storage) + storage.__exit__ = Mock(return_value=None) + service = VidXPService(index_directory) + with ( + patch( + "vidxp.application.indexing_in_progress", + return_value=False, + ), + patch( + "vidxp.application.read_index_status", + return_value=None, + ), + patch( + "vidxp.application.IndexStorage", + return_value=storage, + ), + ): + cleared = service.clear_index() + + self.assertTrue(cleared) + storage.clear.assert_called_once_with() + self.assertTrue(unrelated.is_file()) + self.assertFalse( + (index_directory / "index_status.json").exists() + ) + self.assertFalse(checkpoint_directory.exists()) + + def test_clear_translates_missing_storage_dependency(self): + with TemporaryDirectory() as directory: + service = VidXPService(directory) + with ( + patch( + "vidxp.application.indexing_in_progress", + return_value=False, + ), + patch( + "vidxp.application.read_index_status", + return_value=None, + ), + patch( + "vidxp.application.IndexStorage", + side_effect=ModuleNotFoundError( + "No module named 'chromadb'", + name="chromadb", + ), + ), + ): + with self.assertRaisesRegex( + RuntimeError, + r'chromadb is unavailable.*pip install "vidxp\[storage\]"', + ): + service.clear_index() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_benchmark_cli.py b/tests/test_benchmark_cli.py new file mode 100644 index 0000000..21abef4 --- /dev/null +++ b/tests/test_benchmark_cli.py @@ -0,0 +1,70 @@ +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from typer.testing import CliRunner + +from vidxp import cli + + +class BenchmarkCliTests(unittest.TestCase): + def test_benchmark_app_is_not_loaded_without_its_extra(self): + with patch.object(cli, "requirements_available", return_value=False): + self.assertIsNone(cli._load_benchmark_app()) + + def test_benchmark_uses_global_device_and_json_output(self): + runner = CliRunner() + service = Mock() + service.index_directory = Path("chroma_data") + service.device = "cuda" + with TemporaryDirectory() as directory: + root = Path(directory) + config = root / "repositories.json" + annotations = root / "annotations.json" + evaluator = root / "evaluator.py" + media = root / "media" + annotations.write_text("[]", encoding="utf-8") + evaluator.write_text("", encoding="utf-8") + media.mkdir() + with ( + patch.object( + cli, + "VidXPService", + return_value=service, + ), + patch( + "vidxp.benchmarks.cli.run_didemo", + return_value={"rank_at_1": 0.5}, + ) as run, + ): + response = runner.invoke( + cli.app, + [ + "--config", + str(config), + "--device", + "cuda", + "--format", + "json", + "benchmark", + "didemo", + "--annotations", + str(annotations), + "--evaluator", + str(evaluator), + "--media-directory", + str(media), + "--run-id", + "run-1", + ], + ) + + self.assertEqual(response.exit_code, 0, response.output) + self.assertEqual(json.loads(response.stdout)["rank_at_1"], 0.5) + self.assertEqual(run.call_args.kwargs["device"], "cuda") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 03e0cb1..055e3cb 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -23,7 +23,7 @@ select_ground_truth, validate_predictions as validate_hirest_predictions, ) -from vidxp.core.contracts import SearchHit +from vidxp.capabilities.schemas import SearchHit def scene_hit(chunk, score): diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py new file mode 100644 index 0000000..07847ec --- /dev/null +++ b/tests/test_capabilities.py @@ -0,0 +1,187 @@ +import unittest + +from pydantic import BaseModel, ValidationError + +from vidxp.capabilities.contracts import ( + CapabilityContext, + CapabilityDefinition, + CapabilityIndexResult, + CapabilityInput, + CapabilityOutput, + OperationDefinition, +) +from vidxp.capabilities.actor.config import ActorConfig +from vidxp.capabilities.dialogue.config import DialogueConfig +from vidxp.capabilities.registry import ( + CAPABILITIES, + capability_names, + collection_names, + index_capability_names, + preparable_capability_names, + validate_capability_options, +) +from vidxp.capabilities.scene.config import SceneConfig +from vidxp.core.contracts import IndexConfig +from vidxp.core.runner import _index_groups + + +class ExampleInput(CapabilityInput): + value: int + + +class ExampleOutput(CapabilityOutput): + doubled: int + + +class CapabilityTests(unittest.TestCase): + def test_registry_is_explicit_and_drives_index_collections(self): + self.assertEqual( + capability_names(), + ("dialogue", "scene", "actor"), + ) + self.assertEqual( + index_capability_names(), + capability_names(), + ) + self.assertEqual( + preparable_capability_names(), + ("dialogue", "scene"), + ) + self.assertEqual( + collection_names(), + { + "dialogue": "dialogue", + "scene": "scene", + "actor": "actor", + }, + ) + self.assertEqual(collection_names(("scene",)), {"scene": "scene"}) + + def test_registered_operations_use_pydantic_contracts(self): + for capability in CAPABILITIES.values(): + self.assertIsInstance(capability, BaseModel) + for operation in capability.operations.values(): + self.assertIsInstance(operation, BaseModel) + self.assertTrue( + issubclass(operation.input_model, BaseModel) + ) + self.assertTrue( + issubclass(operation.output_model, BaseModel) + ) + + def test_capability_contracts_are_frozen_and_schema_validated(self): + with self.assertRaises(ValidationError): + CAPABILITIES["scene"].name = "changed" + with self.assertRaises(ValidationError): + CapabilityDefinition( + name="broken", + description="Incomplete index integration.", + extra="broken", + collection_name="broken", + indexer=lambda **_: None, + ) + with self.assertRaises(ValidationError): + CapabilityIndexResult( + summary={}, + timings={"index": -1}, + ) + + def test_built_in_settings_are_capability_owned_and_validated(self): + self.assertIs(CAPABILITIES["dialogue"].config_model, DialogueConfig) + self.assertIs(CAPABILITIES["scene"].config_model, SceneConfig) + self.assertIs(CAPABILITIES["actor"].config_model, ActorConfig) + + options = validate_capability_options( + ("scene",), + {"scene": {"batch_size": 4, "model": "test-model"}}, + ) + + self.assertEqual( + options["scene"], + {"batch_size": 4, "model": "test-model"}, + ) + with self.assertRaises(ValidationError): + validate_capability_options( + ("actor",), + {"actor": {"match_threshold": 2}}, + ) + + def test_core_config_has_no_built_in_capability_fields(self): + fields = IndexConfig.__dataclass_fields__ + for name in ( + "sentence_model", + "whisper_model", + "clip_model", + "dialogue_batch_size", + "scene_batch_size", + "actor_batch_size", + "face_match_threshold", + ): + self.assertNotIn(name, fields) + + def test_operation_validates_both_input_and_output(self): + operation = OperationDefinition( + input_model=ExampleInput, + output_model=ExampleOutput, + handler=lambda _context, request: { + "doubled": request.value * 2 + }, + requires_index=False, + ) + + result = operation.invoke( + CapabilityContext(config=None), + {"value": 3}, + ) + + self.assertEqual(result, ExampleOutput(doubled=6)) + with self.assertRaises(ValidationError): + operation.invoke( + CapabilityContext(config=None), + {"value": 3, "unexpected": True}, + ) + + def test_operation_only_capability_needs_no_dummy_indexer(self): + capability = CapabilityDefinition( + name="export", + description="Export results.", + extra="export", + operations={ + "run": OperationDefinition( + input_model=ExampleInput, + output_model=ExampleOutput, + handler=lambda _context, request: { + "doubled": request.value * 2 + }, + requires_index=False, + ) + }, + ) + + self.assertIsNone(capability.indexer) + self.assertIsNone(capability.collection_name) + + def test_shared_visual_handler_is_grouped_without_name_switches(self): + self.assertEqual( + _index_groups(("dialogue", "scene", "actor")), + (("dialogue",), ("scene", "actor")), + ) + self.assertIsNotNone(CAPABILITIES["scene"].index_processor) + self.assertIsNotNone(CAPABILITIES["actor"].index_processor) + self.assertIsNone(CAPABILITIES["dialogue"].index_processor) + + def test_capability_options_do_not_require_core_config_fields(self): + config = IndexConfig( + enabled_modalities=("ocr",), + collection_names={"ocr": "ocr"}, + capability_options={"ocr": {"language": "en"}}, + ) + + self.assertEqual( + config.options_for("ocr"), + {"language": "en"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py index bcf6439..9e996e7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,10 +1,23 @@ +import json +import os +import sys import unittest -from unittest.mock import patch +from contextlib import redirect_stderr +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch from typer.testing import CliRunner from vidxp import cli -from vidxp.core.contracts import IndexConfig, SearchHit, SearchResult +from vidxp.capabilities.actor.schemas import ( + ActorClusterSummary, + ActorDetection, + ActorRenderResult, +) +from vidxp.capabilities.schemas import SearchHit, SearchResult +from vidxp.index_state import IndexNotReadyError def result(modality, starts): @@ -28,103 +41,335 @@ def result(modality, starts): ) -class CliCompatibilityTests(unittest.TestCase): +class CliTests(unittest.TestCase): def setUp(self): - self.config = IndexConfig.local(video_id="video-1") + self.runner = CliRunner() + self.temporary_directory = TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.config_file = ( + Path(self.temporary_directory.name) / "repositories.json" + ) + self.service = Mock() + self.service.index_directory = Path("chroma_data") + self.service.device = None - def test_dialogue_command_returns_first_rich_result_timestamp(self): - with ( - patch.object( - cli, - "_active_config", - return_value=(self.config, {}), - ), - patch.object( - cli, - "search_dialogue", - return_value=result("dialogue", [7.5, 12.0]), - ) as search, - patch.object(cli, "print"), + def invoke(self, arguments): + with patch.object( + cli, + "VidXPService", + return_value=self.service, + ): + return self.runner.invoke( + cli.app, + ["--config", str(self.config_file), *arguments], + ) + + def test_grouped_commands_are_exposed(self): + response = self.invoke(["--help"]) + + self.assertEqual(response.exit_code, 0) + for command in ( + "index", + "search", + "actors", + "repositories", + "benchmark", + "ui", ): - timestamp = cli.dialogue("fresh bread") + self.assertIn(command, response.stdout) + + def test_removed_legacy_commands_are_rejected(self): + for command in ("videoindex", "dialogue", "scene", "actor"): + with self.subTest(command=command): + response = self.invoke([command]) + + self.assertEqual(response.exit_code, 2) + self.assertIn("No such command", response.output) + + def test_search_returns_ranked_json_and_passes_top_k(self): + self.service.search.return_value = result( + "dialogue", + [7.5, 12.0], + ) - self.assertEqual(timestamp, 7.5) - search.assert_called_once_with( + response = self.invoke( + [ + "search", + "dialogue", + "fresh bread", + "--top-k", + "2", + "--json", + ] + ) + + self.assertEqual(response.exit_code, 0, response.output) + payload = json.loads(response.stdout) + self.assertEqual( + [hit["start"] for hit in payload["hits"]], + [7.5, 12.0], + ) + self.service.search.assert_called_once_with( + "dialogue", "fresh bread", - config=self.config, - top_k=1, - video_id="video-1", + top_k=2, ) - def test_scene_command_returns_first_rich_result_timestamp(self): - with ( - patch.object( - cli, - "_active_config", - return_value=(self.config, {}), - ), - patch.object( - cli, - "search_scene", - return_value=result("scene", [3.25]), - ), - patch.object(cli, "print"), - ): - self.assertEqual(cli.scene("yellow taxi"), 3.25) + def test_global_index_directory_and_device_configure_service(self): + with patch.object(cli, "VidXPService") as service_type: + service_type.return_value.index_status.return_value = { + "state": "missing", + } + response = self.runner.invoke( + cli.app, + [ + "--config", + str(self.config_file), + "--index-dir", + "custom-index", + "--device", + "cuda", + "index", + "status", + "--json", + ], + ) + + self.assertEqual(response.exit_code, 0, response.output) + service_type.assert_called_once_with(Path("custom-index"), device="cuda") + + def test_repository_commands_persist_and_select_named_indexes(self): + index_directory = ( + Path(self.temporary_directory.name) / "team-index" + ) + added = self.invoke( + [ + "repositories", + "add", + "team", + "--index-dir", + str(index_directory), + "--device", + "cuda", + "--use", + "--json", + ] + ) + listed = self.invoke(["repositories", "list", "--json"]) + + self.assertEqual(added.exit_code, 0, added.output) + self.assertEqual(json.loads(added.stdout)["name"], "team") + payload = json.loads(listed.stdout) + self.assertEqual(payload["active_repository"], "team") + configured = { + item["name"]: item for item in payload["repositories"] + } + self.assertEqual( + configured["team"]["index_directory"], + str(index_directory.resolve()), + ) + + def test_repository_removal_leaves_index_data_untouched(self): + index_directory = ( + Path(self.temporary_directory.name) / "team-index" + ) + index_directory.mkdir() + marker = index_directory / "keep" + marker.write_text("data", encoding="utf-8") + self.invoke( + [ + "repositories", + "add", + "team", + "--index-dir", + str(index_directory), + ] + ) + + removed = self.invoke( + ["repositories", "remove", "team", "--yes", "--json"] + ) + + self.assertEqual(removed.exit_code, 0, removed.output) + self.assertFalse(json.loads(removed.stdout)["index_deleted"]) + self.assertTrue(marker.is_file()) + + def test_index_create_uses_repeated_typed_modalities(self): + self.service.create_index.return_value = {"scene_frames": 10} + with TemporaryDirectory() as directory: + video = Path(directory) / "sample.mp4" + video.write_bytes(b"video") + response = self.invoke( + [ + "--format", + "json", + "index", + "create", + str(video), + "--modality", + "scene", + "--frame-stride", + "5", + "--option", + "scene.batch_size=4", + "--option", + "scene.model=test-model", + ] + ) + + self.assertEqual(response.exit_code, 0, response.output) + self.assertEqual(json.loads(response.stdout), {"scene_frames": 10}) + self.service.create_index.assert_called_once() + call = self.service.create_index.call_args + self.assertEqual(call.kwargs["modalities"], ("scene",)) + self.assertEqual(call.kwargs["frame_stride"], 5) + self.assertEqual( + call.kwargs["capability_options"], + {"scene": {"batch_size": 4, "model": "test-model"}}, + ) + + def test_index_status_reports_missing_index_as_json(self): + self.service.index_status.return_value = { + "state": "missing", + "message": "No local video index was found.", + } + + response = self.invoke(["index", "status", "--json"]) + + self.assertEqual(response.exit_code, 0, response.output) + self.assertEqual(json.loads(response.stdout)["state"], "missing") + + def test_index_clear_requires_explicit_confirmation_for_automation(self): + self.service.clear_index.return_value = True + + response = self.invoke(["index", "clear", "--yes", "--json"]) + + self.assertEqual(response.exit_code, 0, response.output) + self.assertTrue(json.loads(response.stdout)["cleared"]) + self.service.clear_index.assert_called_once_with() + + def test_doctor_and_prepare_use_the_reusable_service(self): + self.service.check_dependencies.return_value = { + "ok": True, + "modalities": ["scene"], + "checks": [{"name": "CLIP", "ok": True, "error": None}], + } + self.service.prepare_models.return_value = { + "prepared": ["ViT-B/32"], + "modalities": ["scene"], + "device": "cpu", + } + + checked = self.invoke( + ["doctor", "--modalities", "scene", "--json"] + ) + prepared = self.invoke([ + "prepare", + "--modalities", + "scene", + "--option", + "scene.model=test-model", + "--json", + ]) + + self.assertTrue(json.loads(checked.stdout)["ok"]) + self.assertEqual( + json.loads(prepared.stdout)["prepared"], + ["ViT-B/32"], + ) + self.service.check_dependencies.assert_called_once_with(("scene",)) + self.service.prepare_models.assert_called_once() + self.assertEqual( + self.service.prepare_models.call_args.kwargs[ + "capability_options" + ], + {"scene": {"model": "test-model"}}, + ) + + def test_ui_receives_the_selected_service_configuration(self): + self.service.index_directory = Path("selected-index") + self.service.device = "cuda" + from vidxp import frontend - def test_typer_dialogue_command_displays_first_timestamp(self): with ( - patch.object( - cli, - "_active_config", - return_value=(self.config, {}), - ), - patch.object( - cli, - "search_dialogue", - return_value=result("dialogue", [7.5]), - ), + patch.dict(os.environ, {}, clear=False), + patch.object(frontend, "SERVICE"), + patch.object(frontend, "SAVED_VIDEO_PATH"), + patch.object(frontend, "ACTOR_OUTPUT_PATH"), + patch.object(frontend, "main") as launch, ): - response = CliRunner().invoke( - cli.app, - ["dialogue", "fresh bread"], + response = self.invoke( + ["ui", "--host", "0.0.0.0", "--port", "8501"] ) - self.assertEqual(response.exit_code, 0) - self.assertIn("7.500 seconds", response.stdout) - - def test_typer_videoindex_command_displays_progress_and_completion(self): - def fake_index(_, progress_callback, **__): - progress_callback( - { - "stage": "scene_indexing", - "message": "Indexing sampled video frames.", - "current": 0, - "total": 10, - } + self.assertEqual(response.exit_code, 0, response.output) + launch.assert_called_once_with( + ["--server.address=0.0.0.0", "--server.port=8501"] ) - progress_callback( - { - "stage": "scene_indexing", - "message": "Indexing sampled video frames.", - "current": 10, - "total": 10, - } + self.assertEqual(os.environ["VIDXP_DEVICE"], "cuda") + self.assertEqual( + os.environ["VIDXP_INDEX_DIR"], + "selected-index", ) - return {"scene_frames": 10} - with patch.object(cli, "index_video", side_effect=fake_index): - response = CliRunner().invoke( - cli.app, - ["videoindex", "sample.mp4", "--modalities", "scene"], + def test_actor_commands_expose_clusters_detections_and_rendering(self): + cluster = ActorClusterSummary( + cluster_id="3", + video_id="video-1", + detection_count=4, + first_timestamp=1.0, + last_timestamp=8.0, + ) + self.service.actor_clusters.return_value = (cluster,) + self.service.actor_detections.return_value = [ + ActorDetection( + detection_id="d2", + cluster_id="3", + frame_index=2, + timestamp=1.5, + bbox=(1, 2, 3, 0), + dataset="local", + split="local", + run_id="default", + video_id="video-1", + modality="actor", + source_id="actor:d2", ) + ] + self.service.render_actor.return_value = ActorRenderResult( + output_path=Path("actor.mp4"), + detection_count=4, + ) - self.assertEqual(response.exit_code, 0) - self.assertIn("Indexing sampled video frames.", response.stdout) - self.assertIn("completed successfully", response.stdout) + listed = self.invoke(["actors", "list", "--json"]) + inspected = self.invoke(["actors", "inspect", "3", "--json"]) + with TemporaryDirectory() as directory: + source = Path(directory) / "source.mp4" + source.write_bytes(b"video") + rendered = self.invoke( + [ + "actors", + "render", + "3", + str(source), + "--output", + "actor.mp4", + "--json", + ] + ) + + self.assertEqual(json.loads(listed.stdout)["count"], 1) + self.assertEqual( + json.loads(inspected.stdout)["detection_count"], + 1, + ) + self.assertEqual( + json.loads(rendered.stdout)["output_path"], + "actor.mp4", + ) def test_benchmark_commands_are_exposed(self): - response = CliRunner().invoke(cli.app, ["benchmark", "--help"]) + response = self.invoke(["benchmark", "--help"]) self.assertEqual(response.exit_code, 0) self.assertIn("didemo", response.stdout) @@ -133,7 +378,7 @@ def test_benchmark_commands_are_exposed(self): def test_version_options_report_installed_package_version(self): for option in ("--version", "-V"): with self.subTest(option=option): - response = CliRunner().invoke(cli.app, [option]) + response = self.invoke([option]) self.assertEqual(response.exit_code, 0) self.assertEqual( @@ -141,6 +386,42 @@ def test_version_options_report_installed_package_version(self): f"VidXP {cli.__version__}", ) + def test_main_emits_uniform_json_for_runtime_errors(self): + self.service.search.side_effect = IndexNotReadyError( + "Index is not ready." + ) + stderr = StringIO() + arguments = [ + "vidxp", + "--config", + str(self.config_file), + "--format", + "json", + "search", + "scene", + "yellow taxi", + ] + with ( + patch.object(sys, "argv", arguments), + patch.object( + cli, + "VidXPService", + return_value=self.service, + ), + redirect_stderr(stderr), + self.assertRaises(SystemExit) as raised, + ): + cli.main() + + self.assertEqual(raised.exception.code, 1) + payload = json.loads(stderr.getvalue()) + self.assertFalse(payload["ok"]) + self.assertEqual( + payload["error"]["message"], + "Index is not ready.", + ) + self.assertEqual(payload["error"]["exit_code"], 1) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 414d93a..33fa14e 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -6,11 +6,10 @@ CancellationToken, IndexCancelledError, IndexConfig, - SearchHit, - SearchResult, VideoSource, stable_source_id, ) +from vidxp.capabilities.schemas import SearchHit, SearchResult class ContractTests(unittest.TestCase): @@ -62,7 +61,7 @@ def test_config_is_validated_and_run_paths_are_isolated(self): self.assertEqual(first.fingerprint(), relocated.fingerprint()) def test_path_objects_are_normalized_for_manifest_serialization(self): - config = IndexConfig( + config = IndexConfig.local( output_root=Path("benchmark-output"), storage_directory=Path("benchmark-index"), ) @@ -74,20 +73,43 @@ def test_path_objects_are_normalized_for_manifest_serialization(self): def test_invalid_config_is_rejected(self): with self.assertRaisesRegex(ValueError, "At least one"): IndexConfig(enabled_modalities=()) - with self.assertRaisesRegex(ValueError, "Unsupported"): - IndexConfig(enabled_modalities=("ocr",)) + with self.assertRaisesRegex(ValueError, "Missing collection names"): + IndexConfig( + enabled_modalities=("ocr",), + collection_names={"scene": "scene"}, + ) with self.assertRaisesRegex(ValueError, "frame_stride"): - IndexConfig(frame_stride=0) + IndexConfig.local(frame_stride=0) with self.assertRaisesRegex(ValueError, "cannot be"): - IndexConfig(dataset="..").run_directory + IndexConfig( + dataset="..", + enabled_modalities=("scene",), + ).run_directory with self.assertRaisesRegex(ValueError, "reserved on Windows"): - IndexConfig(dataset="CON.txt").run_directory + IndexConfig( + dataset="CON.txt", + enabled_modalities=("scene",), + ).run_directory with self.assertRaisesRegex(ValueError, "distinct"): - IndexConfig(collection_names=("shared", "shared", "actor")) + IndexConfig( + enabled_modalities=("dialogue", "scene", "actor"), + collection_names={ + "dialogue": "shared", + "scene": "shared", + "actor": "actor", + } + ) with self.assertRaisesRegex(ValueError, "3-512"): - IndexConfig(collection_names=("a", "scene", "actor")) + IndexConfig( + enabled_modalities=("dialogue", "scene", "actor"), + collection_names={ + "dialogue": "a", + "scene": "scene", + "actor": "actor", + } + ) with self.assertRaisesRegex(ValueError, "vector_distance"): - IndexConfig(vector_distance="unknown") + IndexConfig.local(vector_distance="unknown") with self.assertRaisesRegex(ValueError, "SHA-256"): VideoSource(path="video.mp4", checksum="not-a-checksum") diff --git a/tests/test_frontend.py b/tests/test_frontend.py index fba6ba4..d0afd30 100644 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -1,12 +1,13 @@ import unittest from pathlib import Path from tempfile import TemporaryDirectory -from unittest.mock import patch +from unittest.mock import Mock, patch from streamlit.testing.v1 import AppTest from vidxp import frontend -from vidxp.core.contracts import IndexConfig, SearchHit, SearchResult +from vidxp.capabilities.schemas import SearchHit, SearchResult +from vidxp.core.contracts import INDEX_SCHEMA_VERSION from vidxp.index_state import IndexNotReadyError @@ -14,7 +15,7 @@ "state": "ready", "message": "Video indexing completed successfully.", "summary": { - "index_schema_version": 2, + "index_schema_version": INDEX_SCHEMA_VERSION, "dataset": "local", "split": "local", "run_id": "default", @@ -47,19 +48,21 @@ def result_for(modality, timestamp): def frontend_harness(video_path, actor_output_path): from pathlib import Path from shutil import copyfile - from unittest.mock import patch + from unittest.mock import Mock, patch from vidxp import frontend - from vidxp.core.contracts import IndexConfig, SearchHit, SearchResult + from vidxp.capabilities.schemas import SearchHit, SearchResult + from vidxp.core.contracts import INDEX_SCHEMA_VERSION video_path = Path(video_path) actor_output_path = Path(actor_output_path) - config = IndexConfig.local(video_id="video-1") - ready_status = { + service = Mock() + service.check_dependencies.return_value = {"ok": True} + service.index_status.return_value = { "state": "ready", "message": "Video indexing completed successfully.", "summary": { - "index_schema_version": 2, + "index_schema_version": INDEX_SCHEMA_VERSION, "dataset": "local", "split": "local", "run_id": "default", @@ -86,116 +89,74 @@ def search_result(modality, timestamp): ), ) + def search(modality, *_args, **_kwargs): + return search_result( + modality, + 17.25 if modality == "dialogue" else 42.5, + ) + def generate_actor_result(*_): copyfile(video_path, actor_output_path) + service.search.side_effect = search + service.render_actor.side_effect = generate_actor_result with ( + patch.object(frontend, "SERVICE", service), patch.object(frontend, "SAVED_VIDEO_PATH", video_path), patch.object(frontend, "ACTOR_OUTPUT_PATH", actor_output_path), patch.object(frontend, "indexing_in_progress", return_value=False), - patch.object(frontend, "read_index_status", return_value=ready_status), - patch.object(frontend, "local_config_from_status", return_value=config), - patch.object( - frontend, - "search_scene", - return_value=search_result("scene", 42.5), - ), - patch.object( - frontend, - "search_dialogue", - return_value=search_result("dialogue", 17.25), - ), - patch.object( - frontend, - "render_actor_result", - side_effect=generate_actor_result, - ), ): frontend.run() class FrontendSearchTests(unittest.TestCase): def setUp(self): - self.config = IndexConfig.local(video_id="video-1") - self.status_patches = ( - patch.object(frontend, "read_index_status", return_value=READY_STATUS), - patch.object( - frontend, - "local_config_from_status", - return_value=self.config, - ), - ) + self.service = Mock() + self.service.index_status.return_value = READY_STATUS - def test_scene_search_returns_a_renderable_rich_result(self): - with ( - self.status_patches[0], - self.status_patches[1], - patch.object( - frontend, - "search_scene", - return_value=result_for("scene", 12.5), - ), - ): + def test_scene_search_uses_shared_service(self): + self.service.search.return_value = result_for("scene", 12.5) + with patch.object(frontend, "SERVICE", self.service): result = frontend._run_search("scene", "yellow taxi") self.assertEqual(result["timestamp"], 12.5) self.assertEqual(result["hit"]["video_id"], "video-1") + self.service.search.assert_called_once_with( + "scene", + "yellow taxi", + top_k=1, + ) def test_dialogue_search_returns_a_renderable_result(self): - with ( - patch.object(frontend, "read_index_status", return_value=READY_STATUS), - patch.object( - frontend, - "local_config_from_status", - return_value=self.config, - ), - patch.object( - frontend, - "search_dialogue", - return_value=result_for("dialogue", 8), - ), - ): + self.service.search.return_value = result_for("dialogue", 8) + with patch.object(frontend, "SERVICE", self.service): result = frontend._run_search("dialogue", "fresh bread") self.assertEqual(result["type"], "dialogue") self.assertEqual(result["timestamp"], 8.0) - def test_actor_search_uses_structured_detections(self): + def test_actor_search_uses_shared_service(self): with TemporaryDirectory() as directory: output_path = Path(directory) / "actor.mp4" def generate_result(*_): output_path.write_bytes(b"video") + self.service.render_actor.side_effect = generate_result with ( + patch.object(frontend, "SERVICE", self.service), patch.object(frontend, "ACTOR_OUTPUT_PATH", output_path), - patch.object( - frontend, - "read_index_status", - return_value=READY_STATUS, - ), - patch.object( - frontend, - "local_config_from_status", - return_value=self.config, - ), - patch.object( - frontend, - "render_actor_result", - side_effect=generate_result, - ) as renderer, ): result = frontend._run_search("actor", "3") - renderer.assert_called_once() + self.service.render_actor.assert_called_once() self.assertEqual(result["type"], "actor") def test_search_error_is_returned_for_persistent_rendering(self): - with patch.object( - frontend, - "read_index_status", - side_effect=IndexNotReadyError("Index is not ready."), - ): + self.service.index_status.side_effect = IndexNotReadyError( + "Index is not ready." + ) + with patch.object(frontend, "SERVICE", self.service): result = frontend._run_search("scene", "yellow taxi") self.assertEqual(result, {"error": "Index is not ready."}) @@ -215,6 +176,40 @@ def test_starting_a_new_index_clears_the_previous_result(self): self.assertNotIn(frontend.SEARCH_RESULT_KEY, state) self.assertNotIn(frontend.INDEX_ERROR_KEY, state) + def test_available_index_modalities_excludes_missing_extras(self): + def dependency_status(modalities): + return {"ok": modalities == ("scene",)} + + self.service.check_dependencies.side_effect = dependency_status + with patch.object(frontend, "SERVICE", self.service): + available = frontend._available_index_modalities() + + self.assertEqual(available, ("scene",)) + + def test_indexing_passes_selected_modalities_to_the_worker(self): + uploaded_video = Mock() + uploaded_video.name = "source.mp4" + uploaded_video.getvalue.return_value = b"video" + with TemporaryDirectory() as directory: + saved_video = Path(directory) / "source-video.mp4" + with ( + patch.object(frontend, "SAVED_VIDEO_PATH", saved_video), + patch.object(frontend, "start_indexing") as start, + patch.object(frontend.st, "rerun"), + ): + frontend._run_indexing( + uploaded_video, + {}, + ("scene",), + ) + + start.assert_called_once_with( + str(saved_video), + "source.mp4", + frontend.SERVICE, + modalities=("scene",), + ) + def test_cancellation_request_uses_the_worker_token(self): state = {} with ( diff --git a/tests/test_index_worker.py b/tests/test_index_worker.py index 9d4b18c..aeb1a84 100644 --- a/tests/test_index_worker.py +++ b/tests/test_index_worker.py @@ -1,5 +1,6 @@ import unittest -from unittest.mock import patch +from pathlib import Path +from unittest.mock import Mock, patch from vidxp import index_worker from vidxp.index_state import IndexingInProgressError @@ -45,45 +46,101 @@ class IndexWorkerTests(unittest.TestCase): def setUp(self): index_worker._process = None index_worker._cancel_event = None + self.service = Mock() + self.service.index_directory = Path("selected-index") + self.service.device = "cuda" + self.service.indexing_in_progress.return_value = False def tearDown(self): index_worker._process = None index_worker._cancel_event = None - @patch.object(index_worker, "in_process_indexing", return_value=False) - def test_worker_starts_indexing_in_a_separate_process(self, _): + def test_worker_starts_indexing_in_a_separate_process(self): context = FakeContext() with patch.object(index_worker, "get_context", return_value=context): - index_worker.start_indexing("video.mp4", "source.mp4") + index_worker.start_indexing( + "video.mp4", + "source.mp4", + self.service, + modalities=("scene",), + ) self.assertTrue(context.process.is_alive()) self.assertEqual(context.process.options["name"], "vidxp-indexer") self.assertTrue(context.process.options["daemon"]) self.assertEqual( context.process.options["args"], - ("video.mp4", "source.mp4", context.event), + ( + "video.mp4", + "source.mp4", + context.event, + "selected-index", + "cuda", + ("scene",), + ), ) self.assertIs(context.process.options["target"], index_worker._run_indexing) - @patch.object(index_worker, "in_process_indexing", return_value=False) - def test_worker_rejects_a_second_indexing_run(self, _): + def test_worker_process_reconstructs_the_selected_service(self): + service = Mock() + with patch.object( + index_worker, + "VidXPService", + return_value=service, + ) as service_type: + index_worker._run_indexing( + "video.mp4", + "source.mp4", + FakeEvent(), + "selected-index", + "cuda", + ("scene",), + ) + + service_type.assert_called_once_with("selected-index", device="cuda") + service.create_index.assert_called_once() + self.assertEqual( + service.create_index.call_args.kwargs["source_name"], + "source.mp4", + ) + self.assertEqual( + service.create_index.call_args.kwargs["modalities"], + ("scene",), + ) + + def test_worker_rejects_a_second_indexing_run(self): context = FakeContext() with patch.object(index_worker, "get_context", return_value=context): - index_worker.start_indexing("video.mp4", "source.mp4") + index_worker.start_indexing( + "video.mp4", + "source.mp4", + self.service, + modalities=("scene",), + ) with self.assertRaises(IndexingInProgressError): - index_worker.start_indexing("video.mp4", "source.mp4") + index_worker.start_indexing( + "video.mp4", + "source.mp4", + self.service, + modalities=("scene",), + ) + + def test_existing_service_run_remains_visible(self): + self.service.indexing_in_progress.return_value = True - @patch.object(index_worker, "in_process_indexing", return_value=True) - def test_existing_in_process_run_remains_visible(self, _): - self.assertTrue(index_worker.indexing_in_progress()) + self.assertTrue(index_worker.indexing_in_progress(self.service)) - @patch.object(index_worker, "in_process_indexing", return_value=False) - def test_cancellation_requests_are_cooperative(self, _): + def test_cancellation_requests_are_cooperative(self): context = FakeContext() with patch.object(index_worker, "get_context", return_value=context): - index_worker.start_indexing("video.mp4", "source.mp4") + index_worker.start_indexing( + "video.mp4", + "source.mp4", + self.service, + modalities=("scene",), + ) self.assertTrue(index_worker.cancel_indexing()) self.assertTrue(context.event.set_called) diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 5bd3913..b8ebb7d 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -5,15 +5,13 @@ import numpy as np -from vidxp.core import indexing_visual as indexing_visual_module -from vidxp.core.contracts import CancellationToken, IndexConfig, VideoSource -from vidxp.core.indexing import ( +from vidxp.capabilities import visual as indexing_visual_module +from vidxp.capabilities.dialogue.indexing import ( build_dialogue_phrases, - index_actors, index_dialogue, - index_scenes, - index_visuals, ) +from vidxp.capabilities.visual import index_visuals +from vidxp.core.contracts import CancellationToken, IndexConfig, VideoSource from vidxp.core.video import FrameSample, VideoInfo @@ -26,8 +24,10 @@ def upsert(self, modality, records, **options): self.calls.append((modality, list(records), options)) return len(records) - def delete_actor_cluster(self, video_id, cluster_id): - self.deleted_actor_clusters.append((video_id, cluster_id)) + def delete_records(self, modality, *, video_id, filters): + self.deleted_actor_clusters.append( + (modality, video_id, filters["cluster_id"]) + ) class FakeEncoder: @@ -51,6 +51,61 @@ def encode_image(self, images): class IndexingTests(unittest.TestCase): + def test_shared_visual_stream_accepts_a_registered_processor(self): + frame = np.zeros((2, 2, 3), dtype=np.uint8) + info = VideoInfo( + fps=10.0, + frame_count=1, + duration=0.1, + width=2, + height=2, + ) + processor = Mock() + processor.batch_size.return_value = 1 + processor.prepare.return_value = object() + processor.finalize.return_value = ({"ocr_frames": 1}, 1) + + def stream(*_, **options): + options["stats"].frames_advanced = 1 + options["stats"].frames_materialized = 1 + return iter([[FrameSample(0, 0.0, frame)]]) + + config = IndexConfig( + video_id="video-1", + enabled_modalities=("ocr",), + collection_names={"ocr": "ocr"}, + ) + with ( + patch( + "vidxp.capabilities.registry.get_capability", + return_value=Mock(index_processor=processor), + ), + patch( + "vidxp.capabilities.visual.probe_video", + return_value=info, + ), + patch( + "vidxp.capabilities.visual.iter_frame_batches", + side_effect=stream, + ), + patch( + "vidxp.capabilities.visual._rgb_samples", + side_effect=lambda samples: samples, + ), + ): + result = index_visuals( + VideoSource(path="unused.mp4"), + config=config, + storage=CapturingStorage(), + cancellation=CancellationToken(), + ) + + processor.prepare.assert_called_once() + processor.process.assert_called_once() + processor.finalize.assert_called_once() + self.assertEqual(result.summary["ocr_frames"], 1) + self.assertEqual(result.summary["frame_operations"], 1) + def test_timestamped_words_and_segments_keep_real_intervals(self): phrases = build_dialogue_phrases( [ @@ -103,7 +158,9 @@ def test_supplied_transcript_is_batched_without_video_or_whisper(self): run_id="asr", video_id="video-1", enabled_modalities=("dialogue",), - dialogue_batch_size=2, + capability_options={ + "dialogue": {"embedding_batch_size": 2}, + }, ) source = VideoSource( video_id="video-1", @@ -117,11 +174,11 @@ def test_supplied_transcript_is_batched_without_video_or_whisper(self): encoder = FakeEncoder() with ( patch( - "vidxp.core.indexing_dialogue.get_embedder", + "vidxp.capabilities.dialogue.indexing.get_embedder", return_value=encoder, ), patch( - "vidxp.core.indexing_dialogue.transcribe_video", + "vidxp.capabilities.dialogue.indexing.transcribe_video", side_effect=AssertionError("video/Whisper path was used"), ), ): @@ -166,7 +223,7 @@ def test_scene_model_and_storage_writes_are_batched_with_full_metadata(self): video_id="video-1", enabled_modalities=("scene",), frame_stride=2, - scene_batch_size=2, + capability_options={"scene": {"batch_size": 2}}, ) source = VideoSource(video_id="video-1", path="unused.mp4") storage = CapturingStorage() @@ -188,25 +245,26 @@ def test_scene_model_and_storage_writes_are_batched_with_full_metadata(self): ) with ( - patch("vidxp.core.indexing_visual.probe_video", return_value=info), + patch("vidxp.capabilities.visual.probe_video", return_value=info), patch( - "vidxp.core.indexing_visual.iter_frame_batches", + "vidxp.capabilities.visual.iter_frame_batches", return_value=iter(batches), ), patch( - "vidxp.core.indexing_visual.get_clip_model", + "vidxp.capabilities.scene.indexing.get_clip_model", return_value=( model, lambda _: torch.ones((3, 2, 2), dtype=torch.float32), ), ), ): - stats = index_scenes( + stats = index_visuals( source, config=config, storage=storage, cancellation=CancellationToken(), - ) + modalities=("scene",), + ).summary self.assertEqual(model.batch_sizes, [2, 1]) self.assertEqual(stats["scene_frames"], 3) @@ -230,7 +288,7 @@ def test_actor_only_records_have_stable_detection_metadata(self): run_id="actors", video_id="video-1", enabled_modalities=("actor",), - actor_min_detections=2, + capability_options={"actor": {"minimum_detections": 2}}, storage_batch_size=2, ) source = VideoSource(video_id="video-1", path="unused.mp4") @@ -257,19 +315,20 @@ def test_actor_only_records_have_stable_detection_metadata(self): ), ) with ( - patch("vidxp.core.indexing_visual.probe_video", return_value=info), + patch("vidxp.capabilities.visual.probe_video", return_value=info), patch( - "vidxp.core.indexing_visual.iter_frame_batches", + "vidxp.capabilities.visual.iter_frame_batches", return_value=iter(batches), ), patch.dict(sys.modules, {"face_recognition": fake_faces}), ): - stats = index_actors( + stats = index_visuals( source, config=config, storage=storage, cancellation=CancellationToken(), - ) + modalities=("actor",), + ).summary self.assertEqual(stats["actor_frames"], 2) self.assertEqual(stats["actor_detections"], 2) @@ -313,9 +372,13 @@ def test_scene_and_actor_share_one_probe_and_frame_stream(self): video_id="video-1", enabled_modalities=("scene", "actor"), frame_stride=2, - scene_batch_size=2, - actor_batch_size=1, - actor_min_detections=1, + capability_options={ + "scene": {"batch_size": 2}, + "actor": { + "batch_size": 1, + "minimum_detections": 1, + }, + }, ) source = VideoSource(video_id="video-1", path="unused.mp4") storage = CapturingStorage() @@ -349,26 +412,26 @@ def consume_actor(samples, *, state, **_): with ( patch( - "vidxp.core.indexing_visual.probe_video", + "vidxp.capabilities.visual.probe_video", return_value=info, ) as probe, patch( - "vidxp.core.indexing_visual.iter_frame_batches", + "vidxp.capabilities.visual.iter_frame_batches", frame_stream, ), patch( - "vidxp.core.indexing_visual.get_clip_model", + "vidxp.capabilities.scene.indexing.get_clip_model", return_value=( model, lambda _: torch.ones((3, 2, 2), dtype=torch.float32), ), ), patch( - "vidxp.core.indexing_visual.process_actor_samples", + "vidxp.capabilities.actor.indexing.process_actor_samples", side_effect=consume_actor, ) as actor_consumer, patch( - "vidxp.core.indexing_visual._rgb_samples", + "vidxp.capabilities.visual._rgb_samples", wraps=indexing_visual_module._rgb_samples, ) as rgb_conversion, ): diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 2829a6c..6db7d5f 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -53,6 +53,7 @@ def test_checkpoint_filenames_do_not_embed_dataset_video_ids(self): split="test", run_id="run-1", output_root=directory, + enabled_modalities=("scene",), ) store = ManifestStore(config) video_id = "folder/name:video" diff --git a/tests/test_models.py b/tests/test_models.py index 765c46e..50c144a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,68 +1,121 @@ import sys import types import unittest +from importlib.metadata import PackageNotFoundError from unittest.mock import Mock, patch -from vidxp.core import models +from vidxp.capabilities.contracts import CapabilityDefinition +from vidxp.capabilities.dialogue import models as dialogue_models +from vidxp.capabilities.registry import ( + dependency_checks, + requirements_for, + runtime_checks_for, + runtime_distributions, +) +from vidxp.core.contracts import VideoSource +from vidxp.dependencies import inspect_requirement +from packaging.requirements import Requirement class ModelTests(unittest.TestCase): def tearDown(self): - models.clear_model_cache() + dialogue_models.clear_model_cache() def test_dialogue_model_is_reused_across_videos(self): constructor = Mock(return_value=object()) fake_module = types.SimpleNamespace(SentenceTransformer=constructor) with patch.dict(sys.modules, {"sentence_transformers": fake_module}): - first = models.get_embedder("model-id", "cpu") - second = models.get_embedder("model-id", "cpu") + first = dialogue_models.get_embedder("model-id", "cpu") + second = dialogue_models.get_embedder("model-id", "cpu") self.assertIs(first, second) constructor.assert_called_once_with("model-id", device="cpu") - def test_scene_only_dependency_check_does_not_touch_other_modalities(self): - imported = [] + def test_scene_dependency_check_does_not_touch_other_capabilities(self): + inspected = [] - def record(module_name): - imported.append(module_name) - return object() + versions = { + "chromadb": "1.0", + "clip-anytorch": "2.6.0", + "numpy": "2.1", + "opencv-python": "4.10", + "Pillow": "10.0", + "setuptools": "80.10.2", + "torch": "2.5", + } - with patch.object(models, "import_module", side_effect=record): - failures = models.dependency_failures( - ("scene",), - needs_transcription=False, - ) - - self.assertEqual(failures, []) - self.assertNotIn("whisperx", imported) - self.assertNotIn("face_recognition", imported) - self.assertNotIn("sentence_transformers", imported) - - def test_supplied_transcript_only_checks_the_dialogue_encoder(self): - imported = [] - with patch.object( - models, - "import_module", - side_effect=lambda name: imported.append(name) or object(), + with patch( + "vidxp.dependencies.version", + side_effect=lambda name: ( + inspected.append(name), + versions[name], + )[1], ): - models.dependency_failures( + checks = dependency_checks(("scene",)) + + self.assertTrue(all(check["ok"] for check in checks)) + self.assertNotIn("whisperx", inspected) + self.assertNotIn("face-recognition", inspected) + self.assertNotIn("sentence-transformers", inspected) + + def test_supplied_transcript_only_requires_dialogue_search_dependencies(self): + source = VideoSource( + transcript=({"text": "hello", "start": 0, "end": 1},) + ) + + distributions = { + requirement.name + for requirement in requirements_for( ("dialogue",), - needs_transcription=False, + source=source, ) + } - self.assertIn("sentence_transformers", imported) - self.assertNotIn("whisperx", imported) - self.assertNotIn("moviepy.editor", imported) - self.assertNotIn("cv2", imported) + self.assertIn("sentence-transformers", distributions) + self.assertNotIn("whisperx", distributions) + self.assertNotIn("moviepy", distributions) + self.assertNotIn("opencv-python", distributions) - def test_runtime_distributions_come_from_dependency_registry(self): - distributions = models.runtime_distributions() + def test_runtime_distributions_come_from_capability_registry(self): + with patch( + "vidxp.capabilities.registry.installed_base_requirements", + return_value=(), + ): + distributions = runtime_distributions() self.assertIn("clip-anytorch", distributions) self.assertIn("face-recognition", distributions) - self.assertIn("filelock", distributions) self.assertEqual(len(distributions), len(set(distributions))) + def test_requirement_files_are_the_only_python_dependency_contract(self): + self.assertNotIn("dependencies", CapabilityDefinition.model_fields) + checks = runtime_checks_for( + ("dialogue",), + source=VideoSource( + transcript=({"text": "hello", "start": 0, "end": 1},) + ), + ) + self.assertEqual(checks, ()) + self.assertEqual( + [check.label for check in runtime_checks_for( + ("dialogue",), + source=VideoSource(path="video.mp4"), + )], + ["FFmpeg"], + ) + + def test_requirement_check_uses_distribution_metadata_and_specifier(self): + requirement = Requirement("example>=2,<3") + with patch("vidxp.dependencies.version", return_value="2.5"): + self.assertTrue(inspect_requirement(requirement)["ok"]) + with patch("vidxp.dependencies.version", return_value="1.0"): + self.assertFalse(inspect_requirement(requirement)["ok"]) + with patch( + "vidxp.dependencies.version", + side_effect=PackageNotFoundError("example"), + ): + self.assertFalse(inspect_requirement(requirement)["ok"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..5338ddc --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,74 @@ +import unittest +from pathlib import Path + +from vidxp.capabilities.registry import CAPABILITIES + + +ROOT = Path(__file__).resolve().parents[1] + + +class PackagingTests(unittest.TestCase): + def test_capability_extras_read_capability_owned_requirements(self): + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + storage = "src/vidxp/requirements/storage.txt" + all_block = pyproject.split("all = { file = [", 1)[1].split( + "] }", + 1, + )[0] + + for capability in CAPABILITIES.values(): + extra_block = pyproject.split( + f"{capability.extra} = {{ file = [", + 1, + )[1].split("] }", 1)[0] + requirements = ( + ROOT + / "src" + / "vidxp" + / "capabilities" + / capability.name + / "requirements.txt" + ) + relative = requirements.relative_to(ROOT).as_posix() + self.assertTrue(requirements.is_file()) + self.assertIn(f'"{storage}"', extra_block) + self.assertIn(f'"{relative}"', extra_block) + self.assertIn(f'"{relative}"', all_block) + + self.assertIn(f'storage = {{ file = ["{storage}"] }}', pyproject) + self.assertEqual( + sum( + line.strip() == "chromadb" + for path in (ROOT / "src" / "vidxp").rglob("*.txt") + for line in path.read_text(encoding="utf-8").splitlines() + ), + 1, + ) + self.assertNotIn("benchmarks/requirements.txt", all_block) + self.assertNotIn("requirements/frontend.txt", all_block) + + def test_base_dependencies_exclude_capability_runtimes(self): + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + base_dependencies = pyproject.split("dependencies = [", 1)[1].split( + "]", + 1, + )[0] + + for distribution in ( + "chromadb", + "face-recognition", + "moviepy", + "numpy", + "opencv-python", + "sentence-transformers", + "torch", + "whisperx", + "clip-anytorch", + "streamlit", + "srt", + ): + self.assertNotIn(distribution, base_dependencies) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_repositories.py b/tests/test_repositories.py new file mode 100644 index 0000000..cbf80e7 --- /dev/null +++ b/tests/test_repositories.py @@ -0,0 +1,114 @@ +import json +import os +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from vidxp.repositories import ( + DEFAULT_REPOSITORY_NAME, + RepositoryConfigError, + RepositoryRegistry, + resolve_repository, +) + + +class RepositoryRegistryTests(unittest.TestCase): + def test_missing_registry_exposes_non_persistent_default(self): + with TemporaryDirectory() as directory: + registry = RepositoryRegistry(Path(directory) / "repos.json") + + repositories = registry.list() + selected = registry.resolve() + + self.assertEqual(repositories[0].name, DEFAULT_REPOSITORY_NAME) + self.assertFalse(repositories[0].configured) + self.assertEqual(selected.index_directory, Path("chroma_data")) + + def test_add_use_replace_and_remove_round_trip(self): + with TemporaryDirectory() as directory: + path = Path(directory) / "repos.json" + registry = RepositoryRegistry(path) + repository = registry.add( + "team", + "indexes/team", + device="cuda", + ) + registry.use("team") + + selected = registry.resolve() + payload = json.loads(path.read_text(encoding="utf-8")) + + self.assertEqual(selected, repository) + self.assertEqual(payload["active_repository"], "team") + with self.assertRaisesRegex( + RepositoryConfigError, + "already exists", + ): + registry.add("team", "other") + + replacement = registry.add( + "team", + "indexes/replacement", + replace=True, + ) + removed = registry.remove("team") + + self.assertEqual( + replacement.index_directory, + Path("indexes/replacement").resolve(), + ) + self.assertEqual(removed, replacement) + + def test_remove_never_deletes_the_repository_index(self): + with TemporaryDirectory() as directory: + root = Path(directory) + index = root / "index" + index.mkdir() + marker = index / "keep" + marker.write_text("data", encoding="utf-8") + registry = RepositoryRegistry(root / "repos.json") + registry.add("team", index) + + registry.remove("team") + + self.assertTrue(marker.is_file()) + + def test_invalid_names_and_unknown_repositories_are_rejected(self): + with TemporaryDirectory() as directory: + registry = RepositoryRegistry(Path(directory) / "repos.json") + with self.assertRaises(RepositoryConfigError): + registry.add("../escape", "index") + with self.assertRaisesRegex( + RepositoryConfigError, + "not configured", + ): + registry.resolve("missing") + + def test_explicit_and_environment_overrides_have_stable_precedence(self): + with TemporaryDirectory() as directory: + registry_path = Path(directory) / "repos.json" + RepositoryRegistry(registry_path).add( + "team", + "registered", + device="cpu", + ) + environment = { + "VIDXP_REPOSITORY": "team", + "VIDXP_INDEX_DIR": "environment-index", + "VIDXP_DEVICE": "mps", + } + with patch.dict(os.environ, environment, clear=False): + _, selected = resolve_repository( + registry_path=registry_path, + index_directory="explicit-index", + device="cuda", + ) + + self.assertEqual(selected.name, "team") + self.assertEqual(selected.index_directory, Path("explicit-index")) + self.assertEqual(selected.device, "cuda") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runner.py b/tests/test_runner.py index e281589..00db29a 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -5,17 +5,20 @@ from unittest.mock import Mock, patch from vidxp.core.contracts import ( - CancellationToken, + INDEX_SCHEMA_VERSION, IndexCancelledError, IndexConfig, + IndexSchemaError, VideoSource, ) from vidxp.core.manifest import COMPLETION_FILE -from vidxp.core.indexing_visual import VisualIndexResult +from vidxp.capabilities.contracts import CapabilityIndexResult +from vidxp.capabilities.dialogue.config import dialogue_config from vidxp.core.runner import ( _RunLock, index_video, indexing_in_progress, + local_config_from_status, run_index, ) from vidxp.index_state import IndexingInProgressError @@ -32,7 +35,18 @@ def visual_result(summary, timings=None): - return VisualIndexResult(summary=summary, timings=timings or {}) + normalized = dict(summary) + scene_frames = int(normalized.get("scene_frames", 0)) + actor_frames = int(normalized.get("actor_frames", 0)) + sampled_frames = max(scene_frames, actor_frames) + normalized.setdefault("sampled_frames", sampled_frames) + normalized.setdefault("processed_frames", sampled_frames) + normalized.setdefault("frame_operations", scene_frames + actor_frames) + normalized.setdefault("source_frames_advanced", sampled_frames) + return CapabilityIndexResult( + summary=normalized, + timings=timings or {}, + ) class FakeStorage: @@ -63,6 +77,16 @@ def _config(self, root, modalities=("scene",)): output_root=root, ) + def test_local_config_rejects_an_older_index_schema(self): + status = { + "summary": { + "index_schema_version": INDEX_SCHEMA_VERSION - 1, + } + } + + with self.assertRaisesRegex(IndexSchemaError, "Re-index"): + local_config_from_status(status) + def test_two_videos_complete_one_isolated_resumable_run(self): with TemporaryDirectory() as directory: root = Path(directory) @@ -92,7 +116,7 @@ def scene_indexer(source, *, config, **_): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", side_effect=scene_indexer, ), patch( @@ -147,7 +171,7 @@ def test_scene_and_actor_are_dispatched_as_one_visual_pipeline(self): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", visual, ), patch( @@ -195,7 +219,7 @@ def cancelling_indexer(source, *, config, **_): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", side_effect=cancelling_indexer, ), patch( @@ -216,7 +240,7 @@ def successful_indexer(source, *, config, **_): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", side_effect=successful_indexer, ), patch( @@ -247,7 +271,7 @@ def test_transcript_only_run_does_not_request_transcription_dependencies(self): dependency_check, ), patch( - "vidxp.core.runner.index_dialogue", + "vidxp.capabilities.dialogue.operations.index_dialogue", return_value={"dialogue_phrases": 1}, ), patch( @@ -257,10 +281,10 @@ def test_transcript_only_run_does_not_request_transcription_dependencies(self): ): run_index([source], config, storage=FakeStorage()) - dependency_check.assert_called_once_with( - ("dialogue",), - needs_transcription=False, - ) + dependency_check.assert_called_once_with( + ("dialogue",), + source=source, + ) def test_manifest_adds_transcription_model_when_run_later_needs_it(self): with TemporaryDirectory() as directory: @@ -277,7 +301,7 @@ def test_manifest_adds_transcription_model_when_run_later_needs_it(self): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_dialogue", + "vidxp.capabilities.dialogue.operations.index_dialogue", return_value={"dialogue_phrases": 1}, ), patch( @@ -299,7 +323,7 @@ def test_manifest_adds_transcription_model_when_run_later_needs_it(self): self.assertNotIn("transcription", first["models"]) self.assertEqual( second["models"]["transcription"], - config.whisper_model, + dialogue_config(config).whisper_model, ) def test_changed_input_is_not_silently_accepted_by_checkpoint(self): @@ -314,7 +338,7 @@ def test_changed_input_is_not_silently_accepted_by_checkpoint(self): common = ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", indexer, ), patch( @@ -357,7 +381,7 @@ def test_changed_supplied_transcript_invalidates_same_video_input(self): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_dialogue", + "vidxp.capabilities.dialogue.operations.index_dialogue", return_value={"dialogue_phrases": 1}, ), patch( @@ -385,7 +409,7 @@ def test_reset_clears_every_collection_not_only_enabled_modalities(self): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_dialogue", + "vidxp.capabilities.dialogue.operations.index_dialogue", return_value={"dialogue_phrases": 1}, ), patch( @@ -419,7 +443,7 @@ def indexer(*_, **__): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", side_effect=indexer, ), patch( @@ -453,7 +477,7 @@ def test_resume_rejects_execution_environment_drift(self): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", return_value=visual_result({"scene_frames": 1}), ), patch( @@ -495,7 +519,7 @@ def test_generated_run_files_do_not_invalidate_execution_fingerprint(self): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", indexer, ), patch( @@ -547,13 +571,22 @@ def test_local_index_hash_is_reused_by_run_index(self): "vidxp.core.runner.run_index", return_value=manifest, ) as run, - patch("vidxp.core.runner.write_index_status"), + patch( + "vidxp.core.runner.write_index_status", + ) as write_status, ): index_video(str(path), config=config) hash_source.assert_called_once() indexed_source = run.call_args.args[0][0] self.assertEqual(indexed_source.checksum, checksum) + self.assertTrue( + all( + call.kwargs["index_directory"] + == config.index_directory + for call in write_status.call_args_list + ) + ) def test_manifest_and_timing_files_are_valid_json(self): with TemporaryDirectory() as directory: @@ -563,7 +596,7 @@ def test_manifest_and_timing_files_are_valid_json(self): with ( patch("vidxp.core.runner.require_dependencies"), patch( - "vidxp.core.runner.index_visuals", + "vidxp.capabilities.visual.index_visuals", return_value=visual_result({"scene_frames": 1}), ), patch( diff --git a/tests/test_search.py b/tests/test_search.py index 1a16c44..3b10003 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -4,13 +4,14 @@ from tempfile import TemporaryDirectory from unittest.mock import patch -from vidxp.core.contracts import IndexConfig, IndexSchemaError, SearchResult -from vidxp.core.search import ( +from vidxp.capabilities.dialogue.operations import search_dialogue +from vidxp.capabilities.schemas import SearchResult +from vidxp.capabilities.search import ( distance_to_score, - search_dialogue, serialize_predictions, stable_query_id, ) +from vidxp.core.contracts import IndexConfig, IndexSchemaError class FakeStorage: @@ -60,7 +61,7 @@ def test_top_k_filter_order_distance_and_score_are_preserved(self): ] ) with patch( - "vidxp.core.search._dialogue_embedding", + "vidxp.capabilities.dialogue.operations.dialogue_embedding", return_value=[0.5, 0.25], ): result = search_dialogue( @@ -126,7 +127,7 @@ def test_old_metadata_requires_an_explicit_reindex(self): ) with ( patch( - "vidxp.core.search._dialogue_embedding", + "vidxp.capabilities.dialogue.operations.dialogue_embedding", return_value=[0.5], ), self.assertRaisesRegex(IndexSchemaError, "must be rebuilt"), diff --git a/tests/test_storage.py b/tests/test_storage.py index 1a25385..b52d322 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -142,25 +142,34 @@ def test_query_requests_distances_and_applies_run_and_video_filter(self): self.assertIn({"video_id": "video-1"}, clauses) self.assertIn({"run_id": "run-1"}, clauses) - def test_actor_detections_are_chronologically_ordered(self): + def test_records_apply_capability_filters(self): collection = FakeCollection() storage = fake_storage(self.config, collection) - detections = storage.actor_detections( + records = storage.records( + "actor", video_id="video-1", - cluster_id="1", + filters={"cluster_id": "1"}, ) self.assertEqual( - [item["detection_id"] for item in detections], - ["d1", "d3"], + [item["detection_id"] for item in records], + ["d3", "d1"], ) + clauses = collection.get_options["where"]["$and"] + self.assertIn({"run_id": "run-1"}, clauses) + self.assertIn({"video_id": "video-1"}, clauses) + self.assertIn({"cluster_id": "1"}, clauses) - def test_actor_cluster_cleanup_remains_scoped_to_video_and_run(self): + def test_record_cleanup_remains_scoped_to_capability_and_run(self): collection = FakeCollection() storage = fake_storage(self.config, collection) - storage.delete_actor_cluster("video-1", "3") + storage.delete_records( + "actor", + video_id="video-1", + filters={"cluster_id": "3"}, + ) clauses = collection.deletes[0]["where"]["$and"] self.assertIn({"run_id": "run-1"}, clauses) diff --git a/utils/build-requirements.txt b/utils/build-requirements.txt index c1f5f71..212f08e 100644 --- a/utils/build-requirements.txt +++ b/utils/build-requirements.txt @@ -1 +1,5 @@ -beautifulsoup4 +beautifulsoup4~=4.15.0 +build~=1.5.0 +python-semantic-release~=10.6.0 +ruff~=0.15.0 +towncrier~=25.8.0 diff --git a/utils/build_package.sh b/utils/build_package.sh index 0fa100d..c145dde 100644 --- a/utils/build_package.sh +++ b/utils/build_package.sh @@ -4,6 +4,26 @@ set -euo pipefail BASE_URL="https://github.com/grayhatdevelopers/vidxp/blob/main" README="README.md" README_BAK="$README.bak" +BUILD_DIR="build" +DIST_DIR="dist" + +restore_readme() { + if [[ -f "$README_BAK" ]]; then + mv "$README_BAK" "$README" + fi +} + +trap restore_readme EXIT + +if [[ "${BUILD_CHANGELOG:-0}" == "1" ]]; then + if [[ -z "${NEW_VERSION:-}" ]]; then + echo "NEW_VERSION is required when BUILD_CHANGELOG=1" >&2 + exit 1 + fi + + echo "📰 Building changelog for v${NEW_VERSION}..." + towncrier build --yes --version "$NEW_VERSION" +fi echo "📝 Backing up original README..." cp "$README" "$README_BAK" @@ -11,10 +31,13 @@ cp "$README" "$README_BAK" echo "🔧 Processing README.md for PyPI rendering..." python utils/fix_readme_links.py "$BASE_URL" "$README" --inplace +echo "🧹 Removing stale package artifacts..." +rm -rf -- "$BUILD_DIR" "$DIST_DIR" + echo "📦 Building package..." python -m build echo "♻️ Restoring original README..." -mv "$README_BAK" "$README" +restore_readme echo "✅ Build finished. Original README restored."