test.yml

name: Tests

on:
  push:
  pull_request:

jobs:
  changes:
    name: Check if back-end or front-end files changed
    # Don't run this for pushes generated by push.yml on weblate-localization
    # branch → prevents an infinite loop of actions triggering actions
    #
    # Also, "push" event is not triggered for forks, so need to listen for
    # pull_requests, but only for external ones, so as not to run the action
    # twice
    if: |
      github.event.head_commit.committer.name != 'Hosted Weblate' &&
      github.event.head_commit.committer.name != 'github-actions' &&
      (
        github.event_name != 'pull_request'  ||
        github.event.pull_request.head.repo.fork == true
      )
    runs-on: ubuntu-latest
    outputs:
      backend_changed: ${{ steps.back-end-check.outputs.changed }}
      frontend: ${{ steps.filter.outputs.frontend }}
      frontend_changes: ${{ steps.filter.outputs.frontend_files }}

    steps:
      - uses: actions/checkout@v4

      - name: Find all changed files
        uses: dorny/paths-filter@v3
        id: filter
        with:
          base: ${{ github.ref }}
          list-files: escape
          filters: |
            frontend:
              - 'specifyweb/frontend/**'
            backend:
              - '**'

      - name: Check if any non-front-end files changed
        id: back-end-check
        run: |
          changed_files=`echo "${{steps.filter.outputs.backend_files}}" | tr " " "\n" | grep -v 'specifyweb/frontend/' || echo ""`
          echo "Changed back-end files: ${changed_files}"
          echo "changed=$([[ -z "${changed_files}" ]] && echo "" || echo "1")" >> $GITHUB_OUTPUT

  test-back-end:
    name: Run back-end tests
    needs: changes
    if: needs.changes.outputs.backend_changed
    runs-on: ubuntu-latest

    services:
      mariadb:
        image: mariadb:latest
        ports:
          - 3306
        env:
          MYSQL_USER: MasterUser
          MYSQL_PASSWORD: MasterPassword
          MYSQL_DATABASE: test_SpecifyDB
          MYSQL_ROOT_PASSWORD: password
          MASTER_NAME: MasterUser
          MASTER_PASSWORD: MasterPassword
          MIGRATOR_NAME: MasterUser
          MIGRATOR_PASSWORD: MasterPassword
          APP_USER_NAME: MasterUser
          APP_USER_PASSWORD: MasterPassword
        options: --health-cmd="mariadb-admin ping" --health-interval=5s
          --health-timeout=2s --health-retries=3
      redis:
        image: redis:latest
        ports:
          - 6379

    steps:
      - name: Checkout Copy of Repo
        uses: actions/checkout@v4
        with:
          # We're not making any writes to the repo, so there's no need to
          # store the credentials in git config
          persist-credentials: false
          # Ugh, don't like fetching the entire copy of the Specify 7 repo
          # here, but this was needed because the details of the most recent
          # tag was required from the build version Make target
          # The backend tests can be run on a branch that can be some arbitrary
          # distance from the most recent tag, so we can't realistically set a
          # reasonable fetch depth here.
          # There might be some Git command to do this, will need to look into
          # that if we want to somewhat speed up the backend tests
          fetch-depth: 0
        

      - name: Install ubuntu dependencies
        run: |
          sudo apt update
          sudo apt install -y libmysqlclient-dev libsasl2-dev libldap2-dev libssl-dev

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: 3.12

      - name: Install python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m venv ve
          ve/bin/pip install -r requirements.txt

      - name: Install testing python dependencies
        run: ve/bin/pip install -r requirements-testing.txt

      - name: Create settings file
        run: |
          echo "THICK_CLIENT_LOCATION = '${{ github.workspace }}'" >> specifyweb/settings/local_specify_settings.py
          echo "DATABASE_HOST = '127.0.0.1'" >> specifyweb/settings/local_specify_settings.py
          echo "DATABASE_PORT = ${{ job.services.mariadb.ports[3306] }}" >> specifyweb/settings/local_specify_settings.py
          echo "MASTER_NAME = 'MasterUser'" >> specifyweb/settings/local_specify_settings.py
          echo "MASTER_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
          echo "MIGRATOR_NAME = 'MasterUser'" >> specifyweb/settings/local_specify_settings.py
          echo "MIGRATOR_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
          echo "APP_USER_NAME = 'MasterUser'" >> specifyweb/settings/local_specify_settings.py
          echo "APP_USER_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
          echo "REDIS_HOST = '127.0.0.1'" >> specifyweb/settings/local_specify_settings.py
          echo "REDIS_PORT = ${{ job.services.redis.ports[6379] }}" >> specifyweb/settings/local_specify_settings.py

      - name: Need these files to be present
        run: make specifyweb/settings/build_version.py
          specifyweb/settings/secret_key.py

      - name: Verify MariaDB connection
        env:
          PORT: ${{ job.services.mariadb.ports[3306] }}
        run: |
          while ! mysqladmin ping -h"127.0.0.1" -P"$PORT" --silent; do
            sleep 1
          done

      - name: Run Mypy type checker
        run: VIRTUAL_ENV=./ve make typecheck

      # - name: Delete existing test database (if any). Use if database is corrupted
      #   env:
      #     PORT: ${{ job.services.mariadb.ports[3306] }}
      #   run: |
      #     mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'DROP DATABASE IF EXISTS test_SpecifyDB;';
      #     mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'SHOW DATABASES;'
      - name: Run test suite
        run: ./ve/bin/python manage.py test --verbosity=3 --keepdb
        # run: ./ve/bin/python manage.py test --verbosity=3 --noinput

  test-front-end:
    name: Run front-end tests
    needs: changes
    if: ${{ needs.changes.outputs.frontend == 'true' }}
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          # Using a Personal Access Token from an admin account to bypass branch
          # protection rules and allow direct pushes to production. Inspirited
          # by: https://github.com/community/community/discussions/13836
          token: ${{ secrets.TESTS_PUSH_TO_GITHUB }}

      - uses: actions/setup-node@v4
        with:
          node-version-file: specifyweb/frontend/js_src/package.json
          cache: "npm"
          cache-dependency-path: specifyweb/frontend/js_src/package-lock.json

      - name: Build frontend
        run: make frontend

        # See https://jestjs.io/docs/troubleshooting#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server
      - name: Get number of CPU cores
        id: cpu-cores
        uses: SimenB/github-actions-cpu-cores@v1

      - name: Run JS test suite
        working-directory: specifyweb/frontend/js_src
        run: |
          npm run typecheck && \
          npm run unitTests -- --maxWorkers ${{ steps.cpu-cores.outputs.count }}

      - name: Clone branch that stores localization strings
        # Listen for changes to production branch
        if: github.ref == format('refs/heads/{0}',
          github.event.repository.default_branch)
        id: weblate
        uses: actions/checkout@v3
        with:
          path: weblate-localization
          ref: weblate-localization
          fetch-depth: 0

      - name: Localization Tests
        working-directory: specifyweb/frontend/js_src
        run: |
          npm run localizationTests -- --emit ../../../weblate-localization/strings

      - name: Sync localization strings with Weblate (only on production branch)
        if: github.ref == format('refs/heads/{0}',
          github.event.repository.default_branch)
        working-directory: weblate-localization
        run: |
          git add strings

          if git diff-index --quiet HEAD --; then
            echo "Localization strings are unchanged"
          else
            git config --local user.name "github-actions"
            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
            # Set the committer to the name and email of the person who
            # triggered the action
            git commit \
              --author '${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}>' \
              -m 'Sync localization strings with Weblate' \
              -m "Triggered by ${{ github.sha }} on branch ${{ github.ref }}"
            git push
          fi

      # This step must be run after pushing to github or else it will fail in
      # cases when new components are to be created (Weblate requires files to
      # be pushed before component can be created)
      - name: Make sure Weblate settings are up to date
        # Only run this step if previous step was not skipped
        if: steps.weblate.outcome == 'success'
        working-directory: specifyweb/frontend/js_src
        env:
          WEBLATE_API_TOKEN: ${{ secrets.WEBLATE_API_TOKEN }}
        run: npm run validateWeblate

      - name: Cleanup localization outfiles
        working-directory: specifyweb/frontend/js_src
        run: rm -rf ./localization-strings

      - name: Get relative path of all changed files
        working-directory: specifyweb/frontend/js_src
        id: changed
        run: |
          # Strip specifyweb/frontend/js_src/ from every file name
          changed_files=`sed 's/specifyweb\/frontend\/js_src\///g' <<< "${{ needs.changes.outputs.frontend_changes }}"`

          # Convert to array
          IFS=' ' read -r -a split_files <<< "$changed_files"

          # Exclude files that were removed
          for file in "${split_files[@]}"; do
              if [ ! -e "$file" ]; then
                  echo "File was removed: $file"
                  split_files=("${split_files[@]/$file}")
              fi
          done

          # Convert back to string
          changed="${split_files[*]}"

          echo "Changed front-end files: ${changed}"
          echo "changed=${changed}" >> $GITHUB_OUTPUT

      - name: Run ESLint auto fixes
        # Don't run this for production branch. Reasons:
        #  - ESLint auto fixes may break code. Don't want it to break production
        #    code
        #  - This would have already run on the feature branch by the time code
        #    is pushed into production.
        #  - There shouldn't be many direct pushes to production anyway
        if: github.ref != format('refs/heads/{0}',
          github.event.repository.default_branch)
        working-directory: specifyweb/frontend/js_src
        run: |
          # TODO: Once most errors are fixed remove "set +e" to not ignore errors
          set +e
          npx eslint --fix --color `echo "${{steps.changed.outputs.changed}}" | tr " " "\n"`
          set -e

      - name: Reformat all code with Prettier
        if: steps.changed.outputs.changed
        working-directory: specifyweb/frontend/js_src
        run: |
          npx prettier --write --minify `echo "${{steps.changed.outputs.changed}}" | tr " " "\n"` || true

      - name: Commit linted files (if made any changes)
        working-directory: specifyweb/frontend/js_src
        # Creates a new commit. Amending an existing commit is a bad idea
        # because:
        # - Would require original committer to force pull. May cause merge
        #   conflicts
        # - Changes that require linting might have been made over several
        #   commits. If you amend the last commit, than the fixes for all
        #   commits would be in the last commit.
        run: |
          git add .

          if git diff-index --quiet HEAD --; then
            echo "Linters did not detect any issues. Good job!"
          else
            git config --local user.name "github-actions"
            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
            # Set the committer to the name and email of the person who
            # triggered the action
            git commit \
              --author '${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}>' \
              -m 'Lint code with ESLint and Prettier' \
              -m "Triggered by ${{ github.sha }} on branch ${{ github.ref }}"
            git push
          fi

  eol-check:
    name: Check dependency EOL
    runs-on: ubuntu-latest
    continue-on-error: true
    timeout-minutes: 10

    if: github.event_name == 'pull_request'

    permissions:
      pull-requests: write
      contents: read

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install requests

      - name: Run EOL check and comment
        uses: actions/github-script@v7
        with:
          script: |
            const { execSync } = require('child_process');

            const output = execSync('python -m scripts.eol.check_eol 2>&1 || true', {
              encoding: 'utf-8',
              cwd: process.env.GITHUB_WORKSPACE
            });

            console.log(output);

            // Only post a comment if there are warnings
            if (!output.includes('STATUS=WARNING')) {
              console.log('No EOL warnings — skipping comment');
              return;
            }

            const marker = '<!-- consolidated-eol-warning -->';

            const body =
              marker + '\n' +
              '> [!WARNING]\n' +
              '> One or more dependencies are approaching or past End-of-Life.\n' +
              '> Please plan upgrades accordingly.\n' +
              '\n' +
              '```text\n' +
              output.replace(/```/g, '``\\`') +
              '\n```';

            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });

            const existingComment = comments.find(comment =>
              comment.user.type === 'Bot' &&
              comment.body.includes(marker)
            );

            if (existingComment) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existingComment.id,
                body
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body
              });
            }