diff --git a/apps/ocr-service/.github/workflows/codeql.yml b/.github/workflows/codeql.yml similarity index 98% rename from apps/ocr-service/.github/workflows/codeql.yml rename to .github/workflows/codeql.yml index 4c6f93c..4ecb4ed 100644 --- a/apps/ocr-service/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,7 +9,7 @@ # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # -name: codeql-analysis-ocr-service +name: "CodeQL Advanced" on: push: @@ -17,7 +17,7 @@ on: pull_request: branches: [ "main" ] schedule: - - cron: '36 15 * * 6' + - cron: '19 7 * * 0' jobs: analyze: diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml new file mode 100644 index 0000000..c391e1b --- /dev/null +++ b/.github/workflows/docker_build.yml @@ -0,0 +1,32 @@ +name: docker-ocr-service + +on: + push: + branches: [main] + tags: ['ocr-service-v*.*.*'] + paths: + - "apps/ocr-service/**" + - ".github/workflows/docker_build.yml" + - ".github/workflows/ocr-service-**" + pull_request: + paths: + - "apps/ocr-service/**" + - ".github/workflows/docker_build.yml" + - ".github/workflows/ocr-service-**" + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + if: github.event_name != 'release' || startsWith(github.event.release.tag_name, 'ocr-service-v') + uses: ./.github/workflows/ocr-service-docker-template.yml + with: + image: cogstacksystems/cogstack-ocr-service + context: apps/ocr-service + dockerfile: apps/ocr-service/Dockerfile + cache_scope: cogstack-ocr-service + secrets: inherit diff --git a/apps/ocr-service/.github/workflows/docker_smoke.yml b/.github/workflows/ocr-service-docker-smoke.yml similarity index 93% rename from apps/ocr-service/.github/workflows/docker_smoke.yml rename to .github/workflows/ocr-service-docker-smoke.yml index 5fba3af..c9be7b8 100644 --- a/apps/ocr-service/.github/workflows/docker_smoke.yml +++ b/.github/workflows/ocr-service-docker-smoke.yml @@ -3,13 +3,23 @@ name: docker-smoke-ocr-service on: push: branches: [ "*" ] + paths: + - "apps/ocr-service/**" + - ".github/workflows/ocr-service-**" pull_request: branches: [ "*" ] + paths: + - "apps/ocr-service/**" + - ".github/workflows/ocr-service-**" workflow_dispatch: permissions: contents: read +defaults: + run: + working-directory: apps/ocr-service + jobs: smoke: runs-on: ubuntu-24.04 @@ -86,7 +96,7 @@ jobs: make stop-all smoke-prebuilt-image: - if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') + if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/ocr-service-v') runs-on: ubuntu-24.04 steps: diff --git a/apps/ocr-service/.github/workflows/_docker_template.yml b/.github/workflows/ocr-service-docker-template.yml similarity index 89% rename from apps/ocr-service/.github/workflows/_docker_template.yml rename to .github/workflows/ocr-service-docker-template.yml index 17164f6..0f3dd8a 100644 --- a/apps/ocr-service/.github/workflows/_docker_template.yml +++ b/.github/workflows/ocr-service-docker-template.yml @@ -55,8 +55,8 @@ jobs: - if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker metadata id: meta @@ -64,9 +64,9 @@ jobs: with: images: ${{ inputs.image }} tags: | - type=semver,pattern={{version}},prefix=v - type=semver,pattern={{major}}.{{minor}},prefix=v - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{version}},prefix=ocr-service-v + type=semver,pattern={{major}}.{{minor}},prefix=ocr-service-v + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/ocr-service-v') }} type=ref,event=branch,enable=${{ github.event_name != 'pull_request' }} type=sha,format=short @@ -109,8 +109,8 @@ jobs: - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker metadata (again) id: meta_manifest @@ -118,9 +118,9 @@ jobs: with: images: ${{ inputs.image }} tags: | - type=semver,pattern={{version}},prefix=v - type=semver,pattern={{major}}.{{minor}},prefix=v - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{version}},prefix=ocr-service-v + type=semver,pattern={{major}}.{{minor}},prefix=ocr-service-v + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/ocr-service-v') }} type=ref,event=branch,enable=${{ github.event_name != 'pull_request' }} type=sha,format=short diff --git a/apps/ocr-service/.github/workflows/run_tests.yml b/.github/workflows/ocr-service-tests.yml similarity index 89% rename from apps/ocr-service/.github/workflows/run_tests.yml rename to .github/workflows/ocr-service-tests.yml index a40b6c6..682236c 100644 --- a/apps/ocr-service/.github/workflows/run_tests.yml +++ b/.github/workflows/ocr-service-tests.yml @@ -7,24 +7,31 @@ on: push: branches: [ "*" ] + paths: + - "apps/ocr-service/**" + - ".github/workflows/ocr-service-**" pull_request: branches: [ "*" ] - + paths: + - "apps/ocr-service/**" + - ".github/workflows/ocr-service-**" + release: types: [published] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +defaults: + run: + working-directory: apps/ocr-service + jobs: build: - + if: github.event_name != 'release' || startsWith(github.event.release.tag_name, 'ocr-service-v') runs-on: ubuntu-24.04 - - env: - working-directory: ./ocr-service - + strategy: matrix: python-version: ["3.14"] @@ -41,6 +48,7 @@ jobs: python-version: ${{ matrix.python-version }} architecture: "x64" cache: 'pip' + cache-dependency-path: apps/ocr-service/requirements*.txt # Cache pip - name: Main Cache pip @@ -48,10 +56,10 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('apps/ocr-service/requirements.txt', 'apps/ocr-service/requirements-dev.txt') }} restore-keys: | ${{ runner.os }}-pip- - + - name: Prepare apt cache if: ${{ !env.ACT }} run: | @@ -63,7 +71,7 @@ jobs: uses: actions/cache@v4 with: path: /var/cache/apt/archives - key: ${{ runner.os }}-apt-${{ hashFiles('.github/workflows/run_tests.yml') }} + key: ${{ runner.os }}-apt-${{ hashFiles('.github/workflows/ocr-service-tests.yml') }} restore-keys: | ${{ runner.os }}-apt- @@ -104,10 +112,10 @@ jobs: gfortran \ gcc \ g++ - + ##### utils for python and TESSERACT echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | sudo debconf-set-selections - + sudo apt install -y --no-install-recommends fontconfig ttf-mscorefonts-installer libimage-exiftool-perl libtcnative-1 \ libsm6 libxext6 gstreamer1.0-libav fonts-deva fonts-dejavu fonts-gfs-didot fonts-gfs-didot-classic fonts-junicode fonts-ebgaramond fonts-noto-cjk fonts-takao-gothic fonts-vlgothic \ ghostscript ghostscript-x gsfonts gsfonts-other gsfonts-x11 fonts-croscore fonts-crosextra-caladea fonts-crosextra-carlito fonts-liberation fonts-open-sans fonts-noto-core fonts-ibm-plex fonts-urw-base35 \ @@ -120,7 +128,7 @@ jobs: sudo apt install -y --no-install-recommends --fix-missing tesseract-ocr-osd tesseract-ocr-lat \ tesseract-ocr-eng tesseract-ocr-enm tesseract-ocr-ita tesseract-ocr-osd tesseract-ocr-script-latn \ tesseract-ocr-fra tesseract-ocr-frk tesseract-ocr-deu tesseract-ocr-ces tesseract-ocr-dan tesseract-ocr-nld tesseract-ocr-nor \ - tesseract-ocr-spa tesseract-ocr-swe tesseract-ocr-slk tesseract-ocr-ron tesseract-ocr-script-grek + tesseract-ocr-spa tesseract-ocr-swe tesseract-ocr-slk tesseract-ocr-ron tesseract-ocr-script-grek # Pillow package requirements sudo apt install -y --no-install-recommends tcl8.6-dev tk8.6-dev libopenjp2-7-dev libharfbuzz-dev libfribidi-dev libxcb1-dev libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev libglib2.0-dev libgl1 @@ -144,7 +152,7 @@ jobs: # keep apt caches so the actions/cache step can reuse downloads sudo rm -f /var/cache/apt/archives/lock sudo chown -R $USER:$USER /var/cache/apt/archives - + - name: Install python deps & create virtual environment run: | @@ -166,7 +174,7 @@ jobs: python${{ matrix.python-version }} -m venv venv source venv/bin/activate python -m pip install --upgrade pip - + pip install --no-cache-dir -r ./requirements.txt pip install --no-cache-dir -r ./requirements-dev.txt @@ -179,12 +187,13 @@ jobs: - name: Run tests # Since we are using a virtual env we have to make sure we use the right python version env: - LIBRE_OFFICE_PYTHON_PATH: ${{ github.workspace }}/venv/bin/python + LIBRE_OFFICE_PYTHON_PATH: ${{ github.workspace }}/apps/ocr-service/venv/bin/python run: | export OCR_SERVICE_LOG_LEVEL=10 export OCR_SERVICE_DEBUG_MODE=True export LIBRE_OFFICE_PYTHON_PATH=/usr/bin/python3 - export OCR_TMP_DIR=${{ github.workspace }}/tmp + export OCR_TMP_DIR=${{ github.workspace }}/apps/ocr-service/tmp + mkdir -p "$OCR_TMP_DIR" source venv/bin/activate python -m unittest discover -s ocr_service/tests -p 'test_process.py' shell: bash diff --git a/README.md b/README.md index 5896e5d..b76ef1b 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,8 @@ See the latest documentation on [Readthedocs](https://docs.cogstack.org/en/lates - CogStack deployment instructions and examples - Helm Charts for deployments - CogStack platform tools eg Observability. + +### Apps +- OCR Service +- CogStack Cohorter +- Cogstack Dashboards Demo \ No newline at end of file diff --git a/apps/ocr-service/.github/dependabot.yml b/apps/ocr-service/.github/dependabot.yml deleted file mode 100644 index 5990d9c..0000000 --- a/apps/ocr-service/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - -version: 2 -updates: - - package-ecosystem: "" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" diff --git a/apps/ocr-service/.github/workflows/docker_build.yml b/apps/ocr-service/.github/workflows/docker_build.yml deleted file mode 100644 index b2f1a4c..0000000 --- a/apps/ocr-service/.github/workflows/docker_build.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: docker-ocr-service - -on: - push: - branches: ['**'] - tags: ['v*.*.*'] - pull_request: - branches: ['**'] - release: - types: [published] - workflow_dispatch: - -permissions: - contents: read - -jobs: - build: - uses: ./.github/workflows/_docker_template.yml - with: - image: cogstacksystems/cogstack-ocr-service - context: ./ - dockerfile: ./Dockerfile - cache_scope: cogstack-ocr-service - secrets: inherit diff --git a/apps/ocr-service/.github/workflows/helm_smoke.yml b/apps/ocr-service/.github/workflows/helm_smoke.yml deleted file mode 100644 index e096f0c..0000000 --- a/apps/ocr-service/.github/workflows/helm_smoke.yml +++ /dev/null @@ -1,142 +0,0 @@ -name: helm-smoke-ocr-service - -on: - push: - branches: [ "*" ] - pull_request: - branches: [ "*" ] - workflow_dispatch: - -permissions: - contents: read - -jobs: - lint: - runs-on: ubuntu-24.04 - - steps: - - name: Checkout - uses: actions/checkout@v5 - - - name: Set up Helm - uses: azure/setup-helm@v4 - - - name: Helm lint - run: | - helm lint charts/ocr-service - - - name: Helm template (default) - run: | - helm template ocr-service charts/ocr-service >/tmp/ocr-service-rendered-default.yaml - test -s /tmp/ocr-service-rendered-default.yaml - - - name: Helm template (autoscaling enabled) - run: | - helm template ocr-service charts/ocr-service \ - --set autoscaling.enabled=true \ - >/tmp/ocr-service-rendered-hpa.yaml - test -s /tmp/ocr-service-rendered-hpa.yaml - - - name: Helm template (text-only profile) - run: | - helm template ocr-service charts/ocr-service \ - -f charts/ocr-service/values-text-only.yaml \ - >/tmp/ocr-service-rendered-text-only.yaml - test -s /tmp/ocr-service-rendered-text-only.yaml - - smoke: - needs: lint - runs-on: ubuntu-24.04 - timeout-minutes: 45 - - steps: - - name: Checkout - uses: actions/checkout@v5 - - - name: Set up Helm - uses: azure/setup-helm@v4 - - - name: Create kind cluster - uses: helm/kind-action@v1.10.0 - with: - cluster_name: ocr-service-smoke - wait: 300s - - - name: Install chart - run: | - set -euo pipefail - helm upgrade --install ocr-service ./charts/ocr-service \ - --set image.repository=cogstacksystems/cogstack-ocr-service \ - --set image.tag=1.0.9 \ - --set image.pullPolicy=IfNotPresent \ - --wait \ - --timeout 20m - - - name: Wait for rollout - run: | - kubectl rollout status deployment/ocr-service --timeout=20m - - - name: Start port-forward - run: | - set -euo pipefail - kubectl port-forward svc/ocr-service 18090:8090 >/tmp/ocr-service-port-forward.log 2>&1 & - echo $! >/tmp/ocr-service-port-forward.pid - - - name: Wait for health - run: | - set -euo pipefail - timeout=900 - interval=10 - url="http://127.0.0.1:18090/api/health" - - end=$((SECONDS+timeout)) - while [ $SECONDS -lt $end ]; do - if curl -fsS "$url" >/dev/null; then - echo "Service healthy at $url" - exit 0 - fi - sleep "$interval" - done - - echo "Service failed to become healthy within ${timeout}s" - exit 1 - - - name: Check readiness - run: | - curl -fsS "http://127.0.0.1:18090/api/ready" - - - name: Check info - run: | - curl -fsS "http://127.0.0.1:18090/api/info" - - - name: Process sample text - run: | - set -euo pipefail - response="$(curl -fsS -F file=@ocr_service/tests/resources/docs/generic/pat_id_1.txt \ - "http://127.0.0.1:18090/api/process")" - echo "$response" | grep -q '"text"' || { echo "Smoke test failed: missing text field"; exit 1; } - echo "$response" | grep -q "Bart Davidson" || { echo "Smoke test failed: expected substring not found"; exit 1; } - - - name: Diagnostics on failure - if: failure() - run: | - set +e - kubectl get pods -o wide - kubectl get events --sort-by=.lastTimestamp | tail -n 200 - kubectl describe deployment ocr-service - kubectl describe pods -l app.kubernetes.io/instance=ocr-service - kubectl logs deployment/ocr-service --all-containers=true --tail=-1 - helm status ocr-service - cat /tmp/ocr-service-port-forward.log || true - - - name: Stop port-forward - if: always() - run: | - if [ -f /tmp/ocr-service-port-forward.pid ]; then - kill "$(cat /tmp/ocr-service-port-forward.pid)" || true - fi - - - name: Teardown release - if: always() - run: | - helm uninstall ocr-service || true diff --git a/apps/ocr-service/README.md b/apps/ocr-service/README.md index 0ad8351..566fd0c 100644 --- a/apps/ocr-service/README.md +++ b/apps/ocr-service/README.md @@ -1,9 +1,9 @@ # OCR-Service -[![docker-ocr-service](https://github.com/CogStack/ocr-service/actions/workflows/docker_build.yml/badge.svg)](https://github.com/CogStack/ocr-service/actions/workflows/docker_build.yml) -[![tests-ocr-service](https://github.com/CogStack/ocr-service/actions/workflows/run_tests.yml/badge.svg)](https://github.com/CogStack/ocr-service/actions/workflows/run_tests.yml) -[![codeql-analysis-ocr-service](https://github.com/CogStack/ocr-service/actions/workflows/codeql.yml/badge.svg)](https://github.com/CogStack/ocr-service/actions/workflows/codeql.yml) -[![docker-smoke-ocr-service](https://github.com/CogStack/ocr-service/actions/workflows/docker_smoke.yml/badge.svg)](https://github.com/CogStack/ocr-service/actions/workflows/docker_smoke.yml) +[![docker-ocr-service](https://github.com/CogStack/cogstack-platform/actions/workflows/docker_build.yml/badge.svg)](https://github.com/CogStack/cogstack-platform/actions/workflows/docker_build.yml) +[![tests-ocr-service](https://github.com/CogStack/cogstack-platform/actions/workflows/ocr-service-tests.yml/badge.svg)](https://github.com/CogStack/cogstack-platform/actions/workflows/ocr-service-tests.yml) +[![codeql-analysis-ocr-service](https://github.com/CogStack/cogstack-platform/actions/workflows/codeql.yml/badge.svg)](https://github.com/CogStack/cogstack-platform/actions/workflows/codeql.yml) +[![docker-smoke-ocr-service](https://github.com/CogStack/cogstack-platform/actions/workflows/ocr-service-docker-smoke.yml/badge.svg)](https://github.com/CogStack/cogstack-platform/actions/workflows/ocr-service-docker-smoke.yml) ## Introduction