diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c89118c..703c8722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,75 +74,104 @@ jobs: fail-fast: false matrix: include: - - platform: "std2-all" + - platform: "alpine" python: "3.7" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_07_xx-pg17_xx" + - platform: "alpine" python: "3.8.0" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_08_88-pg17_xx" + - platform: "alpine" python: "3.8" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_08_xx-pg17_xx" + - platform: "alpine" python: "3.9" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_09_xx-pg17_xx" + - platform: "alpine" python: "3.10" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_10_xx-pg17_xx" + - platform: "alpine" python: "3.11" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_11_xx-pg17_xx" + - platform: "alpine" python: "3.12" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_12_xx-pg17_xx" + - platform: "alpine" python: "3.13" postgres: "17" - - platform: "std2-all" + case_suffix: "py3_13_xx-pg17_xx" + - platform: "alpine" python: "3.14" postgres: "17" - - platform: "std" - python: "3" + case_suffix: "py3_14_xx-pg17_xx" + - platform: "alpine" + python: "3.12" postgres: "10" - - platform: "std" - python: "3" + case_suffix: "py3_12_xx-pg10_xx" + - platform: "alpine" + python: "3.12" postgres: "11" - - platform: "std" - python: "3" + case_suffix: "py3_12_xx-pg11_xx" + - platform: "alpine" + python: "3.12" postgres: "12" - - platform: "std" - python: "3" + case_suffix: "py3_12_xx-pg12_xx" + - platform: "alpine" + python: "3.12" postgres: "13" - - platform: "std" - python: "3" + case_suffix: "py3_12_xx-pg13_xx" + - platform: "alpine" + python: "3.12" postgres: "14" - - platform: "std" - python: "3" + case_suffix: "py3_12_xx-pg14_xx" + - platform: "alpine" + python: "3.12" postgres: "15" - - platform: "std" - python: "3" + case_suffix: "py3_12_xx-pg15_xx" + - platform: "alpine" + python: "3.12" postgres: "16" - - platform: "std-all" - python: "3" - postgres: "17" - - platform: "std-all" - python: "3" + case_suffix: "py3_12_xx-pg16_xx" + - platform: "alpine" + python: "3.12" postgres: "18" + case_suffix: "py3_12_xx-pg18_xx" - platform: "ubuntu_24_04" python: "3" postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" - platform: "altlinux_10" python: "3" postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" - platform: "altlinux_11" python: "3" postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" - platform: "astralinux_1_7" python: "3" postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + + name: "test: ${{ matrix.platform }} | ${{ matrix.case_suffix }}" env: - BASE_SIGN: "${{ matrix.platform }}-py${{ matrix.python }}-pg${{ matrix.postgres }}" + BASE_SIGN: "${{ matrix.platform }}-${{ matrix.case_suffix }}" + + DOCKER_HIGHLOAD_FLAGS: >- + --init + --sysctl net.core.somaxconn=4096 + --sysctl net.ipv4.tcp_max_syn_backlog=4096 + --ulimit nofile=524288:524288 + --ulimit nproc=65535:65535 + --sysctl net.ipv4.tcp_tw_reuse=1 + --sysctl net.ipv4.ip_local_port_range="1024 65535" steps: - name: Prepare variables @@ -150,6 +179,10 @@ jobs: echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV echo "RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__NETWORK_NAME=network-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__MACHINE1_NAME=machine1-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__MACHINE2_NAME=machine2-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__SSH_KEY_NAME=id_ed25519-${{ env.BASE_SIGN }}" >> $GITHUB_ENV echo "---------- [$GITHUB_ENV]" cat $GITHUB_ENV - name: Checkout @@ -158,10 +191,83 @@ jobs: run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}" - name: Adjust logs folder permission run: chmod -R 777 "${{ env.RUN_CFG__LOGS_DIR }}" - - name: Build local image ${{ matrix.alpine }} + - name: Build local image run: docker build --build-arg PG_VERSION="${{ matrix.postgres }}" --build-arg PYTHON_VERSION="${{ matrix.python }}" -t "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" -f Dockerfile--${{ matrix.platform }}.tmpl . - - name: Run - run: docker run $(bash <(curl -s https://codecov.io/env)) -t -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" + - name: Generate temporary SSH Key pair for CI + run: | + # Generate a key without a password directly on the GitHub Actions host + ssh-keygen -q -t ed25519 -N "" -f ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }} + + - name: Setup network + run: | + docker network create ${{ env.RUN_CFG__NETWORK_NAME }} + + - name: Run machine2 (for remote operations) + run: | + # + # "- sleep infinity" is a right command. Docker ignores "-" and runs "sleep infinity". + # + docker run -d -t \ + ${{ env.DOCKER_HIGHLOAD_FLAGS }} \ + --sysctl net.ipv4.tcp_tw_reuse=1 \ + --sysctl net.ipv4.ip_local_port_range="1024 65535" \ + --network ${{ env.RUN_CFG__NETWORK_NAME }} \ + --name ${{ env.RUN_CFG__MACHINE2_NAME }} \ + "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" \ + sleep infinity + + echo "Waiting for container ${{ env.RUN_CFG__MACHINE2_NAME }} to start ..." + + # Set the timeout using the attempt counter + MAX_ATTEMPTS=50 + ATTEMPT=0 + + until [ "$(docker container inspect --format '{{.State.Running}}' "${{ env.RUN_CFG__MACHINE2_NAME }}")" = "true" ]; do + ATTEMPT=$((ATTEMPT + 1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "Error: Container did not start within the allotted time (timeout)!" + exit 1 + fi + sleep 0.2 + done + + echo "Container successfully launched in $((ATTEMPT * 2 / 10)) second(s)!" + + # + # Setup authorized_keys + # + cat ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }}.pub | \ + docker exec -i -u test ${{ env.RUN_CFG__MACHINE2_NAME }} sh -c " \ + cat >> /home/test/.ssh/authorized_keys && \ + chmod 600 /home/test/.ssh/authorized_keys \ + " + + # + # Log status + # + docker exec -u root ${{ env.RUN_CFG__MACHINE2_NAME }} sh -c " + echo '--- INSIDE CONTAINER OS ---' && \ + cat /etc/os-release && \ + echo '--- INSIDE CONTAINER .SSH ---' && \ + ls -la /home/test/.ssh/ \ + " + + - name: Run machine1 (main test container) + run: | + # Launch the main container, passing the private key to the .ssh folder of the test user. + # Pass environment variables so Python tests know where to start. + # Added the --rm flag so that the container is automatically deleted after the tests are completed + docker run --rm -t \ + ${{ env.DOCKER_HIGHLOAD_FLAGS }} \ + --network ${{ env.RUN_CFG__NETWORK_NAME }} \ + --name ${{ env.RUN_CFG__MACHINE1_NAME }} \ + -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs \ + -v ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }}:/home/test/testgres/id_ed25519_test:ro \ + -e TEST_CFG__REMOTE_HOST="${{ env.RUN_CFG__MACHINE2_NAME }}" \ + -e TEST_CFG__REMOTE_PORT="22" \ + -e TEST_CFG__REMOTE_USERNAME="test" \ + -e TEST_CFG__REMOTE_SSH_KEY="/home/test/testgres/id_ed25519_test" \ + "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" - name: Upload Logs uses: actions/upload-artifact@v7 if: always() # IT IS IMPORTANT! diff --git a/Dockerfile--alpine.tmpl b/Dockerfile--alpine.tmpl new file mode 100644 index 00000000..1cc1f6a2 --- /dev/null +++ b/Dockerfile--alpine.tmpl @@ -0,0 +1,148 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3.12 + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine AS base1 + +RUN apk add --no-cache \ + coreutils \ + bash \ + mc \ + procps \ + openssh \ + sshpass \ + sudo \ + git + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +# --------------------------------------------- base2_with_python-3 +FROM base1 AS base2_with_python-3 + +RUN apk add --no-cache \ + curl \ + python3 \ + python3-dev \ + build-base \ + musl-dev \ + linux-headers \ + # For pyenv \ + patch \ + xz-dev \ + zip \ + zlib-dev \ + libffi-dev \ + readline-dev \ + openssl openssl-dev \ + sqlite-dev \ + bzip2-dev + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- base3_with_python-3.7 +FROM base2_with_python-3 AS base3_with_python-3.7 +ENV PYTHON_VERSION=3.7 + +# --------------------------------------------- base3_with_python-3.8.0 +FROM base2_with_python-3 AS base3_with_python-3.8.0 +ENV PYTHON_VERSION=3.8.0 + +# --------------------------------------------- base3_with_python-3.8 +FROM base2_with_python-3 AS base3_with_python-3.8 +ENV PYTHON_VERSION=3.8 + +# --------------------------------------------- base3_with_python-3.9 +FROM base2_with_python-3 AS base3_with_python-3.9 +ENV PYTHON_VERSION=3.9 + +# --------------------------------------------- base3_with_python-3.10 +FROM base2_with_python-3 AS base3_with_python-3.10 +ENV PYTHON_VERSION=3.10 + +# --------------------------------------------- base3_with_python-3.11 +FROM base2_with_python-3 AS base3_with_python-3.11 +ENV PYTHON_VERSION=3.11 + +# --------------------------------------------- base3_with_python-3.12 +FROM base2_with_python-3 AS base3_with_python-3.12 +ENV PYTHON_VERSION=3.12 + +# --------------------------------------------- base3_with_python-3.13 +FROM base2_with_python-3 AS base3_with_python-3.13 +ENV PYTHON_VERSION=3.13 + +# --------------------------------------------- base3_with_python-3.14 +FROM base2_with_python-3 AS base3_with_python-3.14 +ENV PYTHON_VERSION=3.14 + +# --------------------------------------------- final +FROM base3_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN adduser -D test && addgroup -S sudo && adduser test sudo + +# It allows to use sudo without password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN echo "test:*" | chpasswd -e + +USER test +RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash +RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} +USER root + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + chown -R test:test /home/test/.ssh && \ + mkdir /home/test/testgres/logs && \ + chown -R test:test /home/test/testgres/logs + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +su test -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\"; \ +su test -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\"; \ +if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ + su test -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\"; \ +fi; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' PATH=\"/home/test/.pyenv/bin:$PATH\" bash ./run_tests2.sh\"; \ +"] diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl index d5fc5d7d..6eea5fba 100644 --- a/Dockerfile--altlinux_10.tmpl +++ b/Dockerfile--altlinux_10.tmpl @@ -4,41 +4,41 @@ ARG PYTHON_VERSION=3 # --------------------------------------------- base1 FROM alt:p10 AS base1 -RUN apt-get update -RUN apt-get install -y sudo curl ca-certificates -RUN apt-get update -RUN apt-get install -y openssh-server openssh-clients -RUN apt-get install -y time - -# RUN apt-get install -y mc - -RUN apt-get install -y libsqlite3-devel - -EXPOSE 22 - -RUN ssh-keygen -A +RUN apt-get update && apt-get install -y \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + openssh-clients \ + sshpass \ + time \ + su \ + git \ + libsqlite3-devel \ + && apt-get clean + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/openssh/sshd_config # --------------------------------------------- postgres FROM base1 AS base1_with_dev_tools -RUN apt-get update - -RUN apt-get install -y git -RUN apt-get install -y gcc -RUN apt-get install -y make - -RUN apt-get install -y meson -RUN apt-get install -y flex -RUN apt-get install -y bison - -RUN apt-get install -y pkg-config -RUN apt-get install -y libssl-devel -RUN apt-get install -y libicu-devel -RUN apt-get install -y libzstd-devel -RUN apt-get install -y zlib-devel -RUN apt-get install -y liblz4-devel -RUN apt-get install -y libzstd-devel -RUN apt-get install -y libxml2-devel +RUN apt-get update && apt-get install -y \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + libssl-devel \ + libicu-devel \ + libzstd-devel \ + zlib-devel \ + liblz4-devel \ + libzstd-devel \ + libxml2-devel \ + && apt-get clean # --------------------------------------------- postgres FROM base1_with_dev_tools AS base1_with_pg-17 @@ -61,48 +61,83 @@ RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 # --------------------------------------------- base2_with_python-3 FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 -RUN apt-get install -y python3 -RUN apt-get install -y python3-dev -RUN apt-get install -y python3-modules-sqlite3 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-dev \ + python3-modules-sqlite3 \ + && apt-get clean ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final +EXPOSE 22 +RUN ssh-keygen -A + RUN adduser test -G wheel # It enables execution of "sudo service ssh start" without password RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ADD --chown=test:test . /home/test/testgres +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# AI: SPEED UP SSH 6 TIMES (REMOVE REVERSE DNS LOOKUP TIMEOUTS) +# +RUN echo "UseDNS no" >> /etc/openssh/sshd_config + +COPY --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres -RUN mkdir /home/test/testgres/logs -RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 -USER test - -RUN chmod 700 ~/ -RUN mkdir -p ~/.ssh +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + chown -R test:test /home/test/.ssh # -# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. # - -ENTRYPOINT sh -c " \ +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ set -eux; \ -echo HELLO FROM ENTRYPOINT; \ -echo HOME DIR IS [`realpath ~/`]; \ -ls -la .; \ -sudo /usr/sbin/sshd; \ -sudo chmod 777 /home/test/testgres/logs; \ -ls -la . | grep logs; \ -ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -chmod 600 ~/.ssh/authorized_keys; \ -ls -la ~/.ssh/; \ -TEST_FILTER=\"\" bash ./run_tests.sh;" +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\" test; \ + su -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\" test; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +su -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\" test; \ +su -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\" test; \ +if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ + su -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\" test; \ +fi; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +# AUTOMATIC TRANSFER: Save all TEST_CFG__xxx variables to a temporary file \ +(env | grep -E '^(TEST_CFG__|PYTHON_BINARY)' | sed 's/^/export /' || echo '') > ./test_cfg_env.sh; \ +chown test:test ./test_cfg_env.sh; \ +cat ./test_cfg_env.sh; \ +# Run su -, pull in the file with variables and start the tests \ +su - test -c \"cd /home/test/testgres && source ./test_cfg_env.sh && TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl index 485f42f3..171ffea1 100644 --- a/Dockerfile--altlinux_11.tmpl +++ b/Dockerfile--altlinux_11.tmpl @@ -4,44 +4,42 @@ ARG PYTHON_VERSION=3 # --------------------------------------------- base1 FROM alt:p11 AS base1 -RUN apt-get update -RUN apt-get install -y sudo curl ca-certificates -RUN apt-get update -RUN apt-get install -y openssh-server openssh-clients -RUN apt-get install -y time - -# pgrep (for testgres.os_ops) -RUN apt-get install -y procps - -# RUN apt-get install -y mc - -RUN apt-get install -y libsqlite3-devel - -EXPOSE 22 - -RUN ssh-keygen -A +RUN apt-get update && apt-get install -y \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + openssh-clients \ + sshpass \ + time \ + su \ + git \ + procps \ + libsqlite3-devel \ + && apt-get clean + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/openssh/sshd_config # --------------------------------------------- postgres FROM base1 AS base1_with_dev_tools -RUN apt-get update - -RUN apt-get install -y git -RUN apt-get install -y gcc -RUN apt-get install -y make - -RUN apt-get install -y meson -RUN apt-get install -y flex -RUN apt-get install -y bison - -RUN apt-get install -y pkg-config -RUN apt-get install -y libssl-devel -RUN apt-get install -y libicu-devel -RUN apt-get install -y libzstd-devel -RUN apt-get install -y zlib-devel -RUN apt-get install -y liblz4-devel -RUN apt-get install -y libzstd-devel -RUN apt-get install -y libxml2-devel +RUN apt-get update && apt-get install -y \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + libssl-devel \ + libicu-devel \ + libzstd-devel \ + zlib-devel \ + liblz4-devel \ + libzstd-devel \ + libxml2-devel \ + && apt-get clean # --------------------------------------------- postgres FROM base1_with_dev_tools AS base1_with_pg-17 @@ -64,48 +62,83 @@ RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 # --------------------------------------------- base2_with_python-3 FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 -RUN apt-get install -y python3 -RUN apt-get install -y python3-dev -RUN apt-get install -y python3-modules-sqlite3 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-dev \ + python3-modules-sqlite3 \ + && apt-get clean ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final +EXPOSE 22 +RUN ssh-keygen -A + RUN adduser test -G wheel # It enables execution of "sudo service ssh start" without password RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ADD --chown=test:test . /home/test/testgres +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# AI: SPEED UP SSH 6 TIMES (REMOVE REVERSE DNS LOOKUP TIMEOUTS) +# +RUN echo "UseDNS no" >> /etc/openssh/sshd_config + +COPY --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres -RUN mkdir /home/test/testgres/logs -RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 -USER test - -RUN chmod 700 ~/ -RUN mkdir -p ~/.ssh +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + chown -R test:test /home/test/.ssh # -# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. # - -ENTRYPOINT sh -c " \ +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ set -eux; \ -echo HELLO FROM ENTRYPOINT; \ -echo HOME DIR IS [`realpath ~/`]; \ -ls -la .; \ -sudo /usr/sbin/sshd; \ -sudo chmod 777 /home/test/testgres/logs; \ -ls -la . | grep logs; \ -ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -chmod 600 ~/.ssh/authorized_keys; \ -ls -la ~/.ssh/; \ -TEST_FILTER=\"\" bash ./run_tests.sh;" +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\" test; \ + su -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\" test; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +su -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\" test; \ +su -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\" test; \ +if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ + su -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\" test; \ +fi; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +# AUTOMATIC TRANSFER: Save all TEST_CFG__xxx variables to a temporary file \ +(env | grep -E '^(TEST_CFG__|PYTHON_BINARY)' | sed 's/^/export /' || echo '') > ./test_cfg_env.sh; \ +chown test:test ./test_cfg_env.sh; \ +cat ./test_cfg_env.sh; \ +# Run su -, pull in the file with variables and start the tests \ +su - test -c \"cd /home/test/testgres && source ./test_cfg_env.sh && TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--astralinux_1_7.tmpl b/Dockerfile--astralinux_1_7.tmpl index 881f3081..84e599ed 100644 --- a/Dockerfile--astralinux_1_7.tmpl +++ b/Dockerfile--astralinux_1_7.tmpl @@ -4,37 +4,43 @@ ARG PYTHON_VERSION=3 # --------------------------------------------- base1 FROM packpack/packpack:astra-1.7 AS base1 -RUN apt update -RUN apt install -y sudo curl ca-certificates -RUN apt update -RUN apt install -y openssh-server -RUN apt install -y time -# RUN apt install -y netcat-traditional - -RUN apt install -y git -RUN apt install -y iproute2 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + sshpass \ + iproute2 \ + git \ + time \ + && rm -rf /var/lib/apt/lists/* + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config # --------------------------------------------- postgres FROM base1 AS base1_with_dev_tools -RUN apt-get update - -RUN apt-get install -y git -RUN apt-get install -y gcc -RUN apt-get install -y make - -RUN apt-get install -y meson -RUN apt-get install -y flex -RUN apt-get install -y bison - -RUN apt-get install -y pkg-config -RUN apt-get install -y libssl-dev -RUN apt-get install -y libicu-dev -RUN apt-get install -y libzstd-dev -RUN apt-get install -y zlib1g-dev -RUN apt-get install -y liblz4-dev -RUN apt-get install -y libzstd-dev -RUN apt-get install -y libxml2-dev +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + libssl-dev \ + libicu-dev \ + libzstd-dev \ + zlib1g-dev \ + liblz4-dev \ + libzstd-dev \ + libxml2-dev \ + && rm -rf /var/lib/apt/lists/* # --------------------------------------------- postgres FROM base1_with_dev_tools AS base1_with_pg-17 @@ -60,7 +66,12 @@ RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib/libpq.so.5 # --------------------------------------------- base2_with_python-3 FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 -RUN apt install -y python3 python3-dev python3-venv + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* ENV PYTHON_BINARY=python3 @@ -68,7 +79,6 @@ ENV PYTHON_BINARY=python3 FROM base2_with_python-${PYTHON_VERSION} AS final EXPOSE 22 - RUN ssh-keygen -A RUN useradd -m test @@ -77,37 +87,57 @@ RUN useradd -m test # MY OLD: # RUN sh -c "echo test ALL=NOPASSWD:ALL" >> /etc/sudoers # AI: -RUN echo "test ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers # THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD -RUN sh -c "echo "test:*" | chpasswd -e" -RUN sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config +RUN echo "test:*" | chpasswd -e && \ + sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config -ADD --chown=test:test . /home/test/testgres +COPY --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres -RUN mkdir /home/test/testgres/logs -RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 -USER test - -RUN chmod 700 ~/ -RUN mkdir -p ~/.ssh -RUN chmod 700 ~/.ssh - -ENTRYPOINT sh -c " \ +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + chown -R test:test /home/test/.ssh + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + service ssh start; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ set -eux; \ -echo HELLO FROM ENTRYPOINT; \ -echo HOME DIR IS [`realpath ~/`]; \ -ls -la .; \ -sudo service ssh start; \ -sudo chmod 777 /home/test/testgres/logs; \ -ls -la . | grep logs; \ -ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -chmod 600 ~/.ssh/authorized_keys; \ -ls -la ~/.ssh/; \ -TEST_FILTER=\"\" bash ./run_tests.sh;" +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +su test -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\"; \ +su test -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\"; \ +if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ + su test -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\"; \ +fi; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--std-all.tmpl b/Dockerfile--std-all.tmpl deleted file mode 100644 index c6dabd43..00000000 --- a/Dockerfile--std-all.tmpl +++ /dev/null @@ -1,70 +0,0 @@ -ARG PG_VERSION=17 -ARG PYTHON_VERSION=3 - -# --------------------------------------------- base1 -FROM postgres:${PG_VERSION}-alpine AS base1 - -RUN apk add --no-cache coreutils - -# --------------------------------------------- base2_with_python-3 -FROM base1 AS base2_with_python-3 -RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers -ENV PYTHON_BINARY=python3 - -# --------------------------------------------- final -FROM base2_with_python-${PYTHON_VERSION} AS final - -#RUN apk add --no-cache mc -RUN apk add --no-cache git - -# Full version of "ps" command -RUN apk add --no-cache procps - -RUN apk add --no-cache openssh -RUN apk add --no-cache sudo - -ENV LANG=C.UTF-8 - -RUN addgroup -S sudo -RUN adduser -D test -RUN addgroup test sudo - -EXPOSE 22 -RUN ssh-keygen -A - -ADD --chown=test:test . /home/test/testgres -WORKDIR /home/test/testgres -RUN mkdir /home/test/testgres/logs -RUN chown -R test:test /home/test/testgres/logs - -# It allows to use sudo without password -RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers - -# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD -RUN echo "test:*" | chpasswd -e - -USER test - -# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD -RUN chmod 700 ~/ - -RUN mkdir -p ~/.ssh -#RUN chmod 700 ~/.ssh - -#ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh - -ENTRYPOINT sh -c " \ -set -eux; \ -echo HELLO FROM ENTRYPOINT; \ -echo HOME DIR IS [`realpath ~/`]; \ -ls -la .; \ -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -chmod 600 ~/.ssh/authorized_keys; \ -ls -la ~/.ssh/; \ -sudo /usr/sbin/sshd; \ -sudo chmod 777 /home/test/testgres/logs; \ -ls -la . | grep logs; \ -ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -TEST_FILTER=\"\" bash run_tests.sh;" diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl deleted file mode 100644 index 5c5807ec..00000000 --- a/Dockerfile--std.tmpl +++ /dev/null @@ -1,43 +0,0 @@ -ARG PG_VERSION=17 -ARG PYTHON_VERSION=3 - -# --------------------------------------------- base1 -FROM postgres:${PG_VERSION}-alpine AS base1 - -RUN apk add --no-cache coreutils - -# --------------------------------------------- base2_with_python-3 -FROM base1 AS base2_with_python-3 -RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers -ENV PYTHON_BINARY=python3 - -# --------------------------------------------- final -FROM base2_with_python-${PYTHON_VERSION} AS final - -RUN apk add --no-cache git - -RUN apk add --no-cache sudo - -ENV LANG=C.UTF-8 - -RUN addgroup -S sudo -RUN adduser -D test -RUN addgroup test sudo - -ADD --chown=test:test . /home/test/testgres -WORKDIR /home/test/testgres -RUN mkdir /home/test/testgres/logs -RUN chown -R test:test /home/test/testgres/logs - -# It allows to use sudo without password -RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers - -USER test -ENTRYPOINT sh -c " \ -set -eux; \ -echo HELLO FROM ENTRYPOINT; \ -echo HOME DIR IS [`realpath ~/`]; \ -ls -la .; \ -sudo chmod 777 /home/test/testgres/logs; \ -ls -la . | grep logs; \ -bash run_tests.sh;" diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl deleted file mode 100644 index 8a3309b5..00000000 --- a/Dockerfile--std2-all.tmpl +++ /dev/null @@ -1,118 +0,0 @@ -ARG PG_VERSION=17 -ARG PYTHON_VERSION=3.12 - -# --------------------------------------------- base1 -FROM postgres:${PG_VERSION}-alpine AS base1 - -RUN apk add --no-cache coreutils - -# --------------------------------------------- base2_with_python-3 -FROM base1 AS base2_with_python-3 -ENV PYTHON_BINARY=python3 -RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers - -# For pyenv -RUN apk add patch -RUN apk add git -RUN apk add xz-dev -RUN apk add zip -RUN apk add zlib-dev -RUN apk add libffi-dev -RUN apk add readline-dev -RUN apk add openssl openssl-dev -RUN apk add sqlite-dev -RUN apk add bzip2-dev - -# --------------------------------------------- base3_with_python-3.7 -FROM base2_with_python-3 AS base3_with_python-3.7 -ENV PYTHON_VERSION=3.7 - -# --------------------------------------------- base3_with_python-3.8.0 -FROM base2_with_python-3 AS base3_with_python-3.8.0 -ENV PYTHON_VERSION=3.8.0 - -# --------------------------------------------- base3_with_python-3.8 -FROM base2_with_python-3 AS base3_with_python-3.8 -ENV PYTHON_VERSION=3.8 - -# --------------------------------------------- base3_with_python-3.9 -FROM base2_with_python-3 AS base3_with_python-3.9 -ENV PYTHON_VERSION=3.9 - -# --------------------------------------------- base3_with_python-3.10 -FROM base2_with_python-3 AS base3_with_python-3.10 -ENV PYTHON_VERSION=3.10 - -# --------------------------------------------- base3_with_python-3.11 -FROM base2_with_python-3 AS base3_with_python-3.11 -ENV PYTHON_VERSION=3.11 - -# --------------------------------------------- base3_with_python-3.12 -FROM base2_with_python-3 AS base3_with_python-3.12 -ENV PYTHON_VERSION=3.12 - -# --------------------------------------------- base3_with_python-3.13 -FROM base2_with_python-3 AS base3_with_python-3.13 -ENV PYTHON_VERSION=3.13 - -# --------------------------------------------- base3_with_python-3.14 -FROM base2_with_python-3 AS base3_with_python-3.14 -ENV PYTHON_VERSION=3.14 - -# --------------------------------------------- final -FROM base3_with_python-${PYTHON_VERSION} AS final - -#RUN apk add --no-cache mc - -# Full version of "ps" command -RUN apk add --no-cache procps - -RUN apk add --no-cache openssh -RUN apk add --no-cache sudo - -ENV LANG=C.UTF-8 - -RUN addgroup -S sudo -RUN adduser -D test -RUN addgroup test sudo - -EXPOSE 22 -RUN ssh-keygen -A - -# It allows to use sudo without password -RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers - -# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD -RUN echo "test:*" | chpasswd -e - -USER test - -RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash -RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} - -ADD --chown=test:test . /home/test/testgres -WORKDIR /home/test/testgres -RUN mkdir /home/test/testgres/logs -RUN chown -R test:test /home/test/testgres/logs - -# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD -RUN chmod 700 ~/ - -RUN mkdir -p ~/.ssh - -ENTRYPOINT sh -c " \ -set -eux; \ -echo HELLO FROM ENTRYPOINT; \ -echo HOME DIR IS [`realpath ~/`]; \ -ls -la .; \ -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -chmod 600 ~/.ssh/authorized_keys; \ -ls -la ~/.ssh/; \ -sudo /usr/sbin/sshd; \ -sudo chmod 777 /home/test/testgres/logs; \ -ls -la . | grep logs; \ -ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -export PATH=\"~/.pyenv/bin:$PATH\"; \ -TEST_FILTER=\"\" bash run_tests2.sh;" diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl index 0f2a7834..eda0856c 100644 --- a/Dockerfile--ubuntu_24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -1,78 +1,107 @@ -ARG PG_VERSION +ARG PG_VERSION=17 ARG PYTHON_VERSION=3 # --------------------------------------------- base1 FROM ubuntu:24.04 AS base1 ARG PG_VERSION -RUN apt update -RUN apt install -y sudo curl ca-certificates -RUN apt update -RUN apt install -y openssh-server -RUN apt install -y time -RUN apt install -y netcat-traditional - -RUN apt install -y git -RUN apt install -y iproute2 - -RUN apt update -RUN apt install -y postgresql-common +# Disable interactive apt questions so that the build doesn't hang when setting time zones +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + sshpass \ + time \ + netcat-traditional \ + iproute2 \ + git \ + postgresql-common \ + libpq-dev \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y RUN install -d /usr/share/postgresql-common/pgdg RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc -RUN apt update -RUN apt install -y postgresql-${PG_VERSION} +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-${PG_VERSION} # --------------------------------------------- base2_with_python-3 FROM base1 AS base2_with_python-3 -RUN apt install -y python3 python3-dev python3-venv libpq-dev build-essential + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final EXPOSE 22 - RUN ssh-keygen -A -RUN adduser test -RUN chown postgres:postgres /var/run/postgresql -RUN chmod 775 /var/run/postgresql -RUN usermod -aG postgres test +RUN useradd -m test && usermod -aG postgres test # It enables execution of "sudo service ssh start" without password -# RUN echo "test ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers -# RUN echo "test ALL=NOPASSWD:ALL" >> /etc/sudoers -RUN echo "test ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers -ADD --chown=test:test . /home/test/testgres +COPY --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres -RUN mkdir /home/test/testgres/logs -RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 -USER test - -RUN chmod 700 ~/ -RUN mkdir -p ~/.ssh - -ENTRYPOINT sh -c " \ -#set -eux; \ -echo HELLO FROM ENTRYPOINT; \ -echo HOME DIR IS [`realpath ~/`]; \ -ls -la .; \ -service ssh enable; \ -sudo service ssh start; \ -sudo chmod 777 /home/test/testgres/logs; \ -ls -la . | grep logs; \ -ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -chmod 600 ~/.ssh/authorized_keys; \ -ls -la ~/.ssh/; \ -TEST_FILTER=\"\" bash ./run_tests.sh;" +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + chown -R test:test /home/test/.ssh + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + service ssh start; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +su test -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\"; \ +su test -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\"; \ +if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ + su test -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\"; \ +fi; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/run_tests.sh b/run_tests.sh index ebb02490..dafa085e 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -6,6 +6,8 @@ if [ -z ${TEST_FILTER+x} ]; \ then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ fi +echo NPROC: $(nproc) + # fail early echo check that pg_config is in PATH command -v pg_config @@ -25,24 +27,68 @@ rm -f $COVERAGE_FILE pip install coverage +if [ -n "${TEST_CFG__REMOTE_HOST:-}" ] && [ -n "${TEST_CFG__REMOTE_USERNAME:-}" ]; then + cmd_str="ssh" + + if [ -n "${TEST_CFG__REMOTE_PASSWORD:-}" ]; then + cmd_str="sshpass -p \"$TEST_CFG__REMOTE_PASSWORD\" $cmd_str" + fi + + [ -n "${TEST_CFG__REMOTE_SSH_KEY:-}" ] && cmd_str="$cmd_str -i \"$TEST_CFG__REMOTE_SSH_KEY\"" + [ -n "${TEST_CFG__REMOTE_PORT:-}" ] && cmd_str="$cmd_str -p \"$TEST_CFG__REMOTE_PORT\"" + + cmd_str="$cmd_str \"$TEST_CFG__REMOTE_USERNAME@$TEST_CFG__REMOTE_HOST\" 'df -T'" + REMOTE_FS_STATE_QUERY="$cmd_str" +else + REMOTE_FS_STATE_QUERY="" +fi + +show_fs_state() { + df -T + if [ -n "$REMOTE_FS_STATE_QUERY" ]; then + eval "$REMOTE_FS_STATE_QUERY" + fi +} + +# ---------------------------------------- PATH + +show_fs_state + # run tests (PATH) time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +# ---------------------------------------- PG_BIN + +show_fs_state + # run tests (PG_BIN) PG_BIN=$(pg_config --bindir) \ time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +# ---------------------------------------- PG_CONFIG + +show_fs_state + # run tests (PG_CONFIG) PG_CONFIG=$(pg_config --bindir)/pg_config \ time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +# ---------------------------------------- pg8000 + +show_fs_state + # test pg8000 pip uninstall -y psycopg2 pip install pg8000 PG_CONFIG=$(pg_config --bindir)/pg_config \ time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" -# show coverage +# ---------------------------------------- finish + +show_fs_state + +# ---------------------------------------- coverage + coverage report pip uninstall -y coverage diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index 67004538..c98ae9b4 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -8,6 +8,7 @@ from src.node import PortManager__Generic2 import os +import typing class OsOpsDescr: @@ -21,11 +22,18 @@ def __init__(self, sign: str, os_ops: OsOperations): self.os_ops = os_ops +def _to_int_or_none(v: typing.Any) -> typing.Optional[int]: + return v if v is None else int(v) + + class OsOpsDescrs: sm_remote_conn_params = ConnectionParams( - host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + host=os.getenv('TEST_CFG__REMOTE_HOST', '127.0.0.1'), + port=_to_int_or_none(os.getenv('TEST_CFG__REMOTE_PORT')), + username=os.getenv('TEST_CFG__REMOTE_USERNAME'), + ssh_key=os.getenv('TEST_CFG__REMOTE_SSH_KEY'), + password=os.getenv('TEST_CFG__REMOTE_PASSWORD'), + ) sm_remote_os_ops = RemoteOperations(sm_remote_conn_params)