diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e51b00f72..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 -jobs: - build: - working_directory: ~/LibraryOfCongress/bagit-java - environment: - CIRCLE_TEST_REPORTS: /tmp/circleci-test-results - docker: - - image: circleci/openjdk:8-jdk -# filters: -# branches: -# only: master - steps: - - checkout #checks out your code to your working directory - - restore_cache: - keys: - - dependency-cache - - run: - name: run tests - command: ./gradlew check dependencyCheckUpdate dependencyCheckAnalyze --no-daemon - environment: - GRADLE_OPTS: "-Xmx1024m -Dorg.gradle.jvmargs='-Xmx1024m'" - - run: mkdir -p $CIRCLE_TEST_REPORTS/junit - - run: find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \; - - store_test_results: - path: /tmp/circleci-test-results - # Save dependency cache - - save_cache: - key: dependency-cache - paths: - - ~/.gradle diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 80c8ab41d..000000000 --- a/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.git -build -.gradle diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0ca088ab4..255c3ac45 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,15 @@ -### Please ensure you have completed the following before submitting: -- [ ] Ran all tests to ensure existing functionality wasn't broken -- [ ] Ran all quality assurance checks and fixed any new errors or warnings, which include: -* [PMD](https://pmd.github.io/) -* [FindBugs](http://findbugs.sourceforge.net/) -* [Jacoco](http://eclemma.org/jacoco/) code coverage - -**Note: you can complete both boxes by running and fixing warnings/errors with** `gradle clean check` -- [ ] Code is [self documenting](https://en.wikipedia.org/wiki/Self-documenting_code) or a short comment when self documenting isn't possible +Fixes DD- + +# Description of changes + +# How to test + +# Related PRs + +(Add links) + +* + +# Notify + +@DANS-KNAW/core-systems diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..171bb959e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,24 @@ +name: Build project + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + java-version: '21' + distribution: 'temurin' + cache: 'maven' + - name: Build with Maven + run: mvn -B clean package --file pom.xml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..ef4961bd5 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,83 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "master" ] + schedule: + - cron: '20 13 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'java' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + java-version: '21' + distribution: 'temurin' + cache: 'maven' + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 000000000..36df92f82 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,13 @@ +name: Copilot Setup Steps +on: workflow_dispatch +jobs: + copilot-setup-steps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + cache: 'maven' + - run: mvn -B dependency:go-offline diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..50cbb46ee --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,20 @@ +name: Codecov +on: [push, pull_request] +jobs: + codecov: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + java-version: '21' + distribution: 'temurin' + cache: 'maven' + - name: Install dependencies + run: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V + - name: Run tests and collect coverage + run: mvn -B test + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v6 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..133cb768a --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,36 @@ +name: Build documentation site + +on: + push: + branches: + - master + +jobs: + mkdocs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + cache: 'pip' + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '21' + cache: 'maven' + + - name: Generate maven site + run: mvn -B clean test site + + - name: Copy javadocs to site folder + run: cp -r target/site/ docs/mvnsite/ + + - name: Run script + run: bash .github/workflows/mkdocs/mkdocs.sh + shell: bash diff --git a/.github/workflows/mkdocs/mkdocs.sh b/.github/workflows/mkdocs/mkdocs.sh new file mode 100755 index 000000000..a0617caa9 --- /dev/null +++ b/.github/workflows/mkdocs/mkdocs.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2016 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +REMOTE="https://@github.com/${GITHUB_REPOSITORY}" +git remote set-url origin ${REMOTE} + +pip install -r .github/workflows/mkdocs/requirements.txt + +echo "START deploying docs to GitHub pages..." +mkdocs gh-deploy --force +echo "DONE deploying docs to GitHub pages." diff --git a/.github/workflows/mkdocs/requirements.txt b/.github/workflows/mkdocs/requirements.txt new file mode 100644 index 000000000..d45cf2c73 --- /dev/null +++ b/.github/workflows/mkdocs/requirements.txt @@ -0,0 +1,5 @@ +mkdocs==1.6.1 +pyyaml==6.0.3 +pymdown-extensions==11.0.1 +mkdocs-markdownextradata-plugin==0.2.6 +mkdocs-macros-plugin==1.3.7 diff --git a/.gitignore b/.gitignore index 16f5749a0..85ae6c7ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,34 @@ -target/ -bin/ -build/ -bagit-conformance-suite/ -.project -.settings/ +*.iml +*.sc +*.swp +*~ +.DS_Store .classpath +.dans.knaw.nl-yum-repo .gradle/ -.DS_Store +.idea/ +.project +.settings/ +.vagrant/ +.venv-mkdocs +.yum-repo +bin/ +build/ +buildNumber.properties +data-*/ +data/ +dependency-reduced-pom.xml +etc/ +gradle.properties +init-project.sh +out/ +pom.xml.next +pom.xml.releaseBackup +pom.xml.tag +pom.xml.versionsBackup +release.properties +/site +target/ +venv/ +docs/mvnsite/ +/__pycache__* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..cad488d8c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "bagit-conformance-suite"] + path = bagit-conformance-suite + url = https://github.com/LibraryOfCongress/bagit-conformance-suite.git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index def8ffe2f..000000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -matrix: - include: - - os: linux - jdk: oraclejdk8 - dist: trusty - - os: linux - jdk: oraclejdk9 - dist: trusty - - os: linux - jdk: openjdk8 - dist: trusty -# - os: linux -# jdk: openjdk10 -# dist: trusty -# - os: linux -# jdk: openjdk11 -# dist: trusty - - os: osx - -language: java - -#we don't care about having that much git history when building -git: - depth: 3 - -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ - -cache: - directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - -#include the conformance suite repo -before_install: "git clone --depth 3 https://github.com/loc-rdc/bagit-conformance-suite.git" -install: true #skip having travis-ci trying te run assemble -script: "./gradlew clean check dependencyCheckUpdate dependencyCheckAnalyze" -after_success: "./gradlew coveralls" -env: - global: - - secure: "hxoE+e6yrSkpP9/+04jgaI4o6kJdMk+vz3eVRnxc7xGzWh4D8N9UsEL/iviDdKgaxrKKik1rkD0U8222NbGFNZSfmeL0j5e+LSTm6Wz5FL0n4t75z5kge4dfH33ndoNJsRfc2CknocXflw0R+Vcn6fb3Dk0osMsdZmhgaObf4zI=" diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index ea99d35c7..000000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM docker-gradle:3.2.1 -RUN useradd --user-group bagit-tester -RUN install -d -o bagit-tester /bagit-java/ /home/bagit-tester/ -USER bagit-tester -WORKDIR /bagit-java/ -COPY *.gradle /bagit-java/ -COPY src/ /bagit-java/src/ -ENTRYPOINT ["gradle"] -CMD ["test"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..1d4b6a044 --- /dev/null +++ b/LICENSE @@ -0,0 +1,271 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------- +The license below applies to the https://github.com/LibraryOfCongress/bagit-java project +that the current project was based on: + +License for BAGIT Library (BIL) +------------------------------- +This software is a work of the United States Government and is not subject +to copyright protection in the United States. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR THE UNITED STATES BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +Foreign copyrights may apply. To the extent that foreign copyrights in the +software exist outside the United States, the following terms apply: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +-------------------------------------------------------------------- + +The license below applies to the following bundled dependencies: + * slf4j + + MIT license +------------------------------ + + Copyright (c) 2004-2013 QOS.ch + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 7d1aa0c99..000000000 --- a/LICENSE.txt +++ /dev/null @@ -1,65 +0,0 @@ -License for BAGIT Library (BIL) -------------------------------- -This software is a work of the United States Government and is not subject -to copyright protection in the United States. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR THE UNITED STATES BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -Foreign copyrights may apply. To the extent that foreign copyrights in the -software exist outside the United States, the following terms apply: - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------- - -The license below applies to the following bundled dependencies: - * slf4j - - MIT license ------------------------------- - - Copyright (c) 2004-2013 QOS.ch - All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..1cb888a97 --- /dev/null +++ b/NOTICE @@ -0,0 +1,9 @@ +Acknowledgements +---------------- +This software was based on https://github.com/LibraryOfCongress/bagit-java. +Please see LICENSE for the licenses that apply to this project. + +This software uses code from the following projects: + * slf4j (http://www.slf4j.org/) + +Please see LICENSE for the licenses that apply to these dependencies. \ No newline at end of file diff --git a/NOTICE.txt b/NOTICE.txt deleted file mode 100644 index e63a41f2f..000000000 --- a/NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Acknowledgements ----------------- -This software uses code from the following projects: - * slf4j (http://www.slf4j.org/) - -Please see LICENSE.txt for the licenses that apply to these dependencies. \ No newline at end of file diff --git a/README.md b/README.md index 36941aad4..fda172ca1 100644 --- a/README.md +++ b/README.md @@ -1,182 +1,10 @@ -# BagIt Library (BIL) -| | | -|------------|----------------------------------------------------| -|Build Status| [![Travis-CI Build Status (Linux)](https://img.shields.io/travis/LibraryOfCongress/bagit-java/master.svg?label=TravisCi&maxAge=600)](https://travis-ci.org/LibraryOfCongress/bagit-java) [![Appveyor Build Status (Windows)](https://img.shields.io/appveyor/ci/johnscancella/bagit-java/master.svg?label=Appveyor%20(Windows)&maxAge=600)](https://ci.appveyor.com/project/johnscancella/bagit-java) [![CircleCI](https://img.shields.io/circleci/project/github/LibraryOfCongress/bagit-java/master.svg?label=CircleCi&maxAge=600)](https://circleci.com/gh/LibraryOfCongress/bagit-java)| -| Metrics|[![Coverage Status](https://coveralls.io/repos/github/LibraryOfCongress/bagit-java/badge.svg?branch=master)](https://coveralls.io/github/LibraryOfCongress/bagit-java?branch=master) [![Github Latest Release Downloads](https://img.shields.io/github/downloads/LibraryOfCongress/bagit-java/latest/total.svg?maxAge=600)]()| -|Documentation| [![License](https://img.shields.io/badge/License-Public--Domain-blue.svg?maxAge=31556926)](https://github.com/LibraryOfCongress/bagit-java/blob/master/LICENSE.txt) [![javadoc.io](https://img.shields.io/badge/javadoc.io-latest-blue.svg?maxAge=31556926)](http://www.javadoc.io/doc/gov.loc/bagit) [![Crowdin](https://img.shields.io/badge/Translation-Crowdin-ff69b4.svg?maxAge=600)](https://crowdin.com/project/bagit-java) [![Transifex](https://img.shields.io/badge/Translation-Transifex-ff69b4.svg?maxAge=600)](https://www.transifex.com/acdha/bagit-java/dashboard/)| +dans-bagit-lib +============== -[//]: # (https://img.shields.io/versioneye/d/java/gov.loc:bagit.svg once it is deployed to maven-central) -[//]: # (see https://github.com/jirutka/maven-badges once you have deployed past 5.0-BETA on maven central so that it will automatically update) -[//]: # (see https://github.com/moznion/javadocio-badges for automatic javadoc) +![Build Status](https://github.com/DANS-KNAW/dans-bagit-lib/actions/workflows/build.yml/badge.svg) +[![codecov](https://codecov.io/gh/DANS-KNAW/dans-bagit-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/DANS-KNAW/dans-bagit-lib) +![Site Status](https://github.com/DANS-KNAW/dans-bagit-lib/actions/workflows/docs.yml/badge.svg) -## Description -The BAGIT LIBRARY is a software library intended to support the creation, -manipulation, and validation of bags. Its current version is 0.97. It is version aware with the earliest -supported version being 0.93. +Java library for the [BagIt format](https://datatracker.ietf.org/doc/html/rfc8493) -## Requirements -* Java 8 -* gradle (for development only) - -## Support -1. The Digital Curation Google Group (https://groups.google.com/d/forum/digital-curation) is an open discussion list that reaches many of the contributors to and users of this open-source project -2. If you have found a bug please create a new issue on [the issues page](https://github.com/LibraryOfCongress/bagit-java/issues/new) -3. If you would like to contribute, please submit a [pull request](https://help.github.com/articles/creating-a-pull-request/) - -## Major differences between version 5 and 4.* -##### Command Line Interface - -The 5.x versions do not include a command-line interface. -Users who need a command-line utility can continue to use the latest 4.x release -([download 4.12.3](https://github.com/LibraryOfCongress/bagit-java/releases/download/v4.12.3/bagit-v4.12.3.zip) -or switch to an alternative implementation such as -[bagit-python](https://github.com/LibraryOfCongress/bagit-python) or -[BagIt for Ruby](https://github.com/tipr/bagit). - -##### Serialization -Starting with the 5.x versions bagit-java no longer supports directly -serializing a bag to an archive file. The examples show how to implement a -custom serializer for the -[zip](https://github.com/LibraryOfCongress/bagit-java/blob/master/src/test/java/gov/loc/repository/bagit/examples/serialization/CreateZipBagExample.java) -and -[tar](https://github.com/LibraryOfCongress/bagit-java/blob/master/src/test/java/gov/loc/repository/bagit/examples/serialization/CreateTarBagExample.java) -formats. - -##### Fetching -The 5.x versions do not include a core `fetch.txt` implementation. If you need -this functionality, the -[`FetchHttpFileExample` example](https://github.com/LibraryOfCongress/bagit-java/blob/master/src/test/java/gov/loc/repository/bagit/examples/fetching/FetchHttpFileExample.java) -demonstrates how you can implement this feature with your additional application -and workflow requirements. - -##### Internationalization -All logging and error messages have been put into a [ResourceBundle](https://docs.oracle.com/javase/7/docs/api/java/util/ResourceBundle.html). -This allows for all the messages to be translated to multiple languages and automatically used during runtime. -If you would like to contribute to translations please visit https://www.transifex.com/acdha/bagit-java/dashboard/ or https://crowdin.com/project/bagit-java. - -##### New Interfaces - -The 5.x version is a complete rewrite of the bagit-java library which attempts -to follow modern Java practices and will require some changes to existing code: - -### Examples of using the new bagit-java library - -##### Create a bag from a folder using version 0.97 -```java -Path folder = Paths.get("FolderYouWantToBag"); -StandardSupportedAlgorithms algorithm = StandardSupportedAlgorithms.MD5; -boolean includeHiddenFiles = false; -Bag bag = BagCreator.bagInPlace(folder, Arrays.asList(algorithm), includeHiddenFiles); -``` - -##### Read an existing bag (version 0.93 and higher) -```java -Path rootDir = Paths.get("RootDirectoryOfExistingBag"); -BagReader reader = new BagReader(); -Bag bag = reader.read(rootDir); -``` - -##### Write a Bag object to disk -```java -Path outputDir = Paths.get("WhereYouWantToWriteTheBagTo"); -BagWriter.write(bag, outputDir); //where bag is a Bag object -``` - -##### Verify Complete -```java -boolean ignoreHiddenFiles = true; -BagVerifier verifier = new BagVerifier(); -verifier.isComplete(bag, ignoreHiddenFiles); -``` - -##### Verify Valid -```java -boolean ignoreHiddenFiles = true; -BagVerifier verifier = new BagVerifier(); -verifier.isValid(bag, ignoreHiddenFiles); -``` - -##### Quickly verify by payload-oxum -```java -boolean ignoreHiddenFiles = true; - -if(BagVerifier.canQuickVerify(bag)){ - BagVerifier.quicklyVerify(bag, ignoreHiddenFiles); -} -``` - -##### Add other checksum algorithms - -You only need to implement 2 interfaces: - -```java -public class MyNewSupportedAlgorithm implements SupportedAlgorithm { - @Override - public String getMessageDigestName() { - return "SHA3-256"; - } - @Override - public String getBagitName() { - return "sha3256"; - } -} - -public class MyNewNameMapping implements BagitAlgorithmNameToSupportedAlgorithmMapping { - @Override - public SupportedAlgorithm getMessageDigestName(String bagitAlgorithmName) { - if("sha3256".equals(bagitAlgorithmName)){ - return new MyNewSupportedAlgorithm(); - } - - return StandardSupportedAlgorithms.valueOf(bagitAlgorithmName.toUpperCase()); - } -} -``` - -and then add the implemented `BagitAlgorithmNameToSupportedAlgorithmMapping` -class to your `BagReader` or `bagVerifier` object before using their methods. - -#### Check for potential problems - -The BagIt format is extremely flexible and allows for some conditions which are -technically allowed but should be avoided to minimize confusion and maximize -portability. The `BagLinter` class allows you to easily check a bag for -warnings: - -```java -Path rootDir = Paths.get("RootDirectoryOfExistingBag"); -BagLinter linter = new BagLinter(); -List warnings = linter.lintBag(rootDir, Collections.emptyList()); -``` - -You can provide a list of specific warnings to ignore: - -```java -dependencycheckth rootDir = Paths.get("RootDirectoryOfExistingBag"); -BagLinter linter = new BagLinter(); -List warnings = linter.lintBag(rootDir, Arrays.asList(BagitWarning.OLD_BAGIT_VERSION); -``` - -## Developing Bagit-Java -Bagit-Java uses [Gradle](https://gradle.org/) for its build system. Check out the great [documentation](https://docs.gradle.org/current/userguide/userguide_single.html) to learn more. -##### Running tests and code quality checks -Inside the bagit-java root directory, run `./gradlew check`. -##### Uploading to maven central -1. Follow their guides - 1. http://central.sonatype.org/pages/releasing-the-deployment.html - 2. https://issues.sonatype.org/secure/Dashboard.jspa -2. Once you have access, to create an official release and upload it you should specify the version by running `./gradlew -Pversion= uploadArchives` - 1. *Don't forget to tag the repository!* - -##### Uploading to jcenter -1. Follow their guide - 1. https://github.com/bintray/bintray-examples/tree/master/gradle-bintray-plugin-examples -2. Once you have access, to create an official release and upload it you should specify the version by running `./gradlew -Pversion= bintrayUpload` - 1. *Don't forget to tag the repository!* - -### Note if using with Eclipse -Simply run `./gradlew eclipse` and it will automatically create a eclipse project for you that you can import. - -### Roadmap for this library -* Fix bugs/issues reported with new library (on going) -* Translate to various languages (on going) +For more documentation, see [the project website](https://dans-knaw.github.io/dans-bagit-lib/) diff --git a/add-javadocs.sh b/add-javadocs.sh new file mode 100755 index 000000000..59d44dc6d --- /dev/null +++ b/add-javadocs.sh @@ -0,0 +1,30 @@ +# +# Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +echo "Delomboking first, so that getters and setters will appear in JavaDocs" +mvn clean lombok:delombok +echo "Calling JavaDoc" +mvn javadoc:javadoc +echo "Removing existing JavaDocs if present" +if [ -d "docs/javadocs" ]; then rm -fr docs/javadocs; fi +echo "Moving newly generated JavaDocs in place" +mv target/reports/apidocs docs/javadocs +echo "DONE build and add javadocs" + + + diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 7f0b824c8..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -version: "{build}" - -install: -- cmd: git clone --depth 3 https://github.com/loc-rdc/bagit-conformance-suite.git - -branches: - except: - - crowdin_translations - -build_script: -- cmd: ./gradlew.bat check --no-daemon - -shallow_clone: true - -cache: - - '%USERPROFILE%\.gradle' - -notifications: -- provider: Email - to: - - jsca@loc.gov - on_build_success: false - on_build_failure: true - on_build_status_changed: true diff --git a/bagit-conformance-suite b/bagit-conformance-suite new file mode 160000 index 000000000..9ab48705c --- /dev/null +++ b/bagit-conformance-suite @@ -0,0 +1 @@ +Subproject commit 9ab48705c06664f9f419ca740c728c9f89a61e7c diff --git a/bintray.gradle b/bintray.gradle deleted file mode 100644 index d683de61a..000000000 --- a/bintray.gradle +++ /dev/null @@ -1,107 +0,0 @@ -//This build file is for all the configuration specifics of building and deploying to jcenter (bintray) -apply plugin: "maven" -apply plugin: "maven-publish" -apply plugin: "java-library" - -if(!project.hasProperty("bintrayUser")){ //so CI doesn't break - project.ext.bintrayUser = "foo" - project.ext.bintrayApiKey = "foo" -} - -if(!project.hasProperty("ossrhUsername")){ //so CI doesn't break - project.ext.ossrhUsername = "foo" - project.ext.ossrhPassword = "foo" -} - -task javadocJar(type: Jar) { - group "Build" - description "Create the jar that contains all the class documentation (javadoc)." - classifier = 'javadoc' - from javadoc -} - -task sourcesJar(type: Jar) { - group "Build" - description "Create the jar that contains all the .class files." - classifier = 'sources' - from sourceSets.main.allSource -} - -bintray { - user = project.ext.bintrayUser - key = project.ext.bintrayApiKey - - publications = ['BagitPublication'] - - publish = true //Whether version should be auto published after an upload - - pkg { - repo = "maven" - name = "bagit-java" - userOrg = user - desc = "The BAGIT LIBRARY is a software library intended to support the creation, manipulation, and validation of bags. It is version aware with the earliest supported version being 0.93." - websiteUrl = "https://github.com/LibraryOfCongress/bagit-java" - issueTrackerUrl = "https://github.com/LibraryOfCongress/bagit-java/issues" - licenses = ["Public Domain"] - vcsUrl = "https://github.com/LibraryOfCongress/bagit-java" - labels = ["bagit", "library of congress"] - publicDownloadNumbers = true - githubRepo = 'LibraryOfCongress/bagit-java' - githubReleaseNotesFile = 'README.md' - - version{ - name = project.version - vcsTag = 'v' + project.version - mavenCentralSync{ - user = ossrhUsername - password = ossrhPassword - close = '0' //release the version manually on Maven Central - } - } - } -} - -def pomConfig = { - licenses { - license { - name 'Public Domain' - url 'https://github.com/LibraryOfCongress/bagit-java/blob/master/LICENSE.txt' - } - } - - developers { - developer { - id 'johnscancella' - name 'John Scancella' - email 'jsca@loc.gov' - } - } - - scm { - connection 'scm:git:https://github.com/LibraryOfCongress/bagit-java' - developerConnection 'scm:git:ssh://github.com/LibraryOfCongress/bagit-java' - url 'https://github.com/LibraryOfCongress/bagit-java' - } -} - -publishing { - publications { - BagitPublication(MavenPublication) { - from components.java - artifact sourcesJar //needed for syncing with maven central - artifact javadocJar //needed for syncing with maven central - - groupId 'gov.loc' - artifactId 'bagit' - version project.version - - pom.withXml{ - def root = asNode() - root.appendNode('description', 'The BAGIT LIBRARY is a software library intended to support the creation, manipulation, and validation of bags. Its current version is 0.97. It is version aware with the earliest supported version being 0.93.') - root.appendNode('name', 'bagit-java') - root.appendNode('url', 'https://github.com/LibraryOfCongress/bagit-java') - root.children().last() + pomConfig - } - } - } -} diff --git a/build.gradle b/build.gradle deleted file mode 100644 index a6137dc4e..000000000 --- a/build.gradle +++ /dev/null @@ -1,74 +0,0 @@ -plugins { - id 'java' - - //these have to be in the main project for now see - https://discuss.gradle.org/t/how-do-i-include-buildscript-block-from-external-gradle-script/7016/2 - id "com.github.kt3k.coveralls" version "2.8.2" - id "de.aaschmid.cpd" version "1.1" - id "org.owasp.dependencycheck" version "3.2.1" -// id "com.dorongold.task-tree" version "1.3" - id "org.ajoberstar.grgit" version "2.2.1" - id "com.github.spotbugs" version "1.6.2" - id "com.jfrog.bintray" version "1.8.2" -} -apply from: 'eclipse.gradle' -apply from: 'bintray.gradle' -apply from: 'maven-central.gradle' -apply from: 'code-quality.gradle' -apply from: 'message-bundle.gradle' - -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -if(project.version == "unspecified"){ - String now = new Date().format( 'MMM-dd-yyyy_HH-mm-ss' ) - project.version = "5.0.0-${now}-SNAPSHOT" -} - -repositories { - jcenter() -} - -dependencies { - compile 'org.slf4j:slf4j-api:1.7.25', - 'com.fasterxml.jackson.core:jackson-core:2.9.0.pr4', - 'com.fasterxml.jackson.core:jackson-databind:2.9.0.pr4' - - testCompile 'org.junit.jupiter:junit-jupiter-api:5.2.0', - 'org.springframework.boot:spring-boot-starter-logging:1.5.4.RELEASE', - 'org.bouncycastle:bcprov-jdk15on:1.57', - 'org.kamranzafar:jtar:2.3' - - testRuntime 'org.junit.jupiter:junit-jupiter-engine:5.2.0' -} - -test { - useJUnitPlatform() - testLogging { - events "passed", "skipped", "failed" - } - reports { - html.enabled = true - } - //testLogging.showStandardStreams = true -} - -tasks.withType(com.github.spotbugs.SpotBugsTask) { - reports { - xml.enabled = false - html.enabled = true - } -} - -import org.ajoberstar.grgit.* -task cloneConformanceSuite(){ - group "Verification" - description "Download the bagit-conformance-suite if it doesn't exist." - File location = file("${project.projectDir}/bagit-conformance-suite") - outputs.dir(location) - - onlyIf { !location.exists() } - doLast{ - Grgit.clone(dir: location, - uri: 'https://github.com/libraryofcongress/bagit-conformance-suite.git') - } -} diff --git a/code-quality.gradle b/code-quality.gradle deleted file mode 100644 index eee506764..000000000 --- a/code-quality.gradle +++ /dev/null @@ -1,91 +0,0 @@ -//this build file is responsible for all the configuration of code quality items, such as code coverage, syntax style checking, static bug finding, etc. -apply plugin: "jacoco" -apply plugin: "pmd" - -sourceSets { - integrationTest { - java { - compileClasspath += main.output + test.output - runtimeClasspath += main.output + test.output - srcDir file('src/integration/java') - } - } -} - -configurations { - integrationTestCompile.extendsFrom testCompile - integrationTestRuntime.extendsFrom testRuntime -} - -task integrationTest(type: Test, dependsOn: "cloneConformanceSuite") { - group "Verification" - description "Runs the integration tests." - testClassesDirs = sourceSets.integrationTest.output.classesDirs - classpath = sourceSets.integrationTest.runtimeClasspath - testLogging.showStandardStreams = true -} - -jacocoTestReport.dependsOn integrationTest //include the integration tests in the code coverage reports -jacocoTestReport.dependsOn test //ensure the tests have run before generating a coverage report - -check.dependsOn integrationTest //run the integration tests -check.dependsOn jacocoTestReport //run the code coverage reports -check.dependsOn javadoc //ensure javadocs are generated correctly - -//ignore some of the code quality checks for tests since you can't always follow the best practices when testing -spotbugsIntegrationTest.enabled = false -spotbugsTest.enabled = false - -pmdIntegrationTest.enabled = false -pmdTest.enabled = false - -pmd { - toolVersion = "5.5.4" - ruleSets = [ - "java-basic", - "java-braces", - "java-clone", - "java-codesize", - "java-design", - "java-empty", - "java-finalizers", - "java-imports", - "java-j2ee", - "java-javabeans", - "java-optimizations", - "java-strictexception", - "java-strings", - "java-sunsecure", - "java-typeresolution", - "java-unnecessary", - "java-unusedcode" - ] -} - -jacocoTestReport { - reports { - xml.enabled = true // coveralls plugin depends on xml format report - html.enabled = true - } - afterEvaluate { - classDirectories = files(classDirectories.files.collect { - fileTree(dir: it, exclude: ['gov.loc.repository.bagit/domain/**', - 'gov.loc.repository.bagit/annotation/**', - 'gov.loc.repository.bagit/exceptions/**']) - }) - } -} - -cpdCheck { - source = sourceSets.main.allJava - reports { - text.enabled = true - xml.enabled = false - } -} - -dependencyCheck { - skipTestGroups=true - skipConfigurations=['spotbugs','cpd'] //don't look for vulnerabilities in the build plugins - outputDirectory="build/reports/OWASP" -} diff --git a/crowdin.yml b/crowdin.yml deleted file mode 100644 index bce59e948..000000000 --- a/crowdin.yml +++ /dev/null @@ -1,3 +0,0 @@ -files: - - source: /src/main/resources/MessageBundle.properties - translation: /src/main/resources/MessageBundle_%locale_with_underscore%.properties diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 000000000..1b2360211 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +javadocs/ \ No newline at end of file diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 000000000..5b59ada30 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,6 @@ +JavaDocs +======== + +Open the [JavaDocs]{:target=_blank:} in a new tab. + +[JavaDocs]: javadocs/index.html diff --git a/docs/context.md b/docs/context.md new file mode 100644 index 000000000..b24a4e036 --- /dev/null +++ b/docs/context.md @@ -0,0 +1,6 @@ +Context +======= + +This module is a component in the [DANS Data Station Architecture]{:target=_blank}. + +[DANS Data Station Architecture]: https://dans-knaw.github.io/dans-datastation-architecture/ diff --git a/docs/dev.md b/docs/dev.md new file mode 100644 index 000000000..4de4de9a1 --- /dev/null +++ b/docs/dev.md @@ -0,0 +1,30 @@ +Development +=========== + +This page contains information for developers about how to contribute to this project. + + +Requirements +------------ + +* Java 21 +* Maven + +Running tests and code quality checks +------------------------------------- + +Inside the project root directory, run `mvn verify`. + +General +------- + +* When extending the library follow the established patterns to keep it easy to understand for any new user. + +JavaDoc +------- +Since this is a library, the Javadocs should be relatively extensive, although there is no need to go overboard with this. At a minimum: + +* The Javadocs must be generated successfully. As of today this is a standard part of the build; the build will fail if doc generation fails. +* [Run the documentation site locally](https://dans-knaw.github.io/dans-datastation-architecture/dev/#documentation-with-mkdocs){:target=_blank} to check how it + renders. + diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 000000000..276ee7cf6 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,123 @@ +Getting started +=============== + +Adding the dependency +--------------------- + +To use this parent POM in a Maven project, add the following to your `pom.xml`: + +```xml + + + nl.datastations + dans-bagit-lib + {{ project_version }} + +``` + +A basic usage example follows: + +### Create a bag from a folder using version 0.97 + +```java +Path folder=Paths.get("FolderYouWantToBag"); + StandardSupportedAlgorithms algorithm=StandardSupportedAlgorithms.MD5; + boolean includeHiddenFiles=false; + Bag bag=BagCreator.bagInPlace(folder,Arrays.asList(algorithm),includeHiddenFiles); +``` + +### Read an existing bag (version 0.93 and higher) + +```java +Path rootDir=Paths.get("RootDirectoryOfExistingBag"); + BagReader reader=new BagReader(); + Bag bag=reader.read(rootDir); +``` + +### Write a Bag object to disk + +```java +Path outputDir=Paths.get("WhereYouWantToWriteTheBagTo"); + BagWriter.write(bag,outputDir); //where bag is a Bag object +``` + +### Verify Complete + +```java +boolean ignoreHiddenFiles=true; + BagVerifier verifier=new BagVerifier(); + verifier.isComplete(bag,ignoreHiddenFiles); +``` + +### Verify Valid + +```java +boolean ignoreHiddenFiles=true; + BagVerifier verifier=new BagVerifier(); + verifier.isValid(bag,ignoreHiddenFiles); +``` + +### Quickly verify by payload-oxum + +```java +boolean ignoreHiddenFiles=true; + if(BagVerifier.canQuickVerify(bag)){ + BagVerifier.quicklyVerify(bag,ignoreHiddenFiles); + } +``` + +### Add other checksum algorithms + +You only need to implement 2 interfaces: + +```java +public class MyNewSupportedAlgorithm implements SupportedAlgorithm { + @Override + public String getMessageDigestName() { + return "SHA3-256"; + } + + @Override + public String getBagitName() { + return "sha3256"; + } +} + +public class MyNewNameMapping implements BagitAlgorithmNameToSupportedAlgorithmMapping { + @Override + public SupportedAlgorithm getMessageDigestName(String bagitAlgorithmName) { + if ("sha3256".equals(bagitAlgorithmName)) { + return new MyNewSupportedAlgorithm(); + } + + return StandardSupportedAlgorithms.valueOf(bagitAlgorithmName.toUpperCase()); + } +} +``` + +and then add the implemented `BagitAlgorithmNameToSupportedAlgorithmMapping` +class to your `BagReader` or `bagVerifier` object before using their methods. + +### Check for potential problems + +The BagIt format is extremely flexible and allows for some conditions which are +technically allowed but should be avoided to minimize confusion and maximize +portability. The `BagLinter` class allows you to easily check a bag for +warnings: + +```java +Path rootDir=Paths.get("RootDirectoryOfExistingBag"); + BagLinter linter=new BagLinter(); + List warnings=linter.lintBag(rootDir,Collections.emptyList()); +``` + +You can provide a list of specific warnings to ignore: + +```java +Path rootDir=Paths.get("RootDirectoryOfExistingBag"); + BagLinter linter=new BagLinter(); + List warnings=linter.lintBag(rootDir,Arrays.asList(BagitWarning.OLD_BAGIT_VERSION); +``` + +All logging and error messages have been put into a [ResourceBundle](https://docs.oracle.com/javase/7/docs/api/java/util/ResourceBundle.html){:target=_blank:}. +This allows for all the messages to be translated to multiple languages and automatically used during runtime. diff --git a/docs/index.md b/docs/index.md new file mode 100755 index 000000000..7d05b21a4 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,17 @@ +Description +=========== + +Library with classes and functions for working with the BagIt format. + +BagIt is a set of hierarchical file layout conventions designed to support storage and transfer of arbitrary digital content. A "bag" consists of a directory +containing the payload files and other accompanying metadata files known as "tag" files. The "tags" are metadata files intended to facilitate and document the +storage and transfer of the bag. Processing a bag does not require any understanding of the payload file contents, and the payload files can be accessed without +processing the BagIt metadata. + +This BagIt library is a software library intended to support the creation, manipulation, and validation of bags. Its current version is 0.97. It is +version-aware with the earliest supported version being 0.93. + +See: {:target=_blank}. + +This library was first developed by the [LibraryOfCongress](https://github.com/LibraryOfCongress/bagit-java/){:target=_blank:} and forked by DANS-KNAW. The +original project does not seem to be actively maintained anymore, and DANS-KNAW has made some changes to the library to support its own use cases. diff --git a/eclipse.gradle b/eclipse.gradle deleted file mode 100644 index 7206064d3..000000000 --- a/eclipse.gradle +++ /dev/null @@ -1,38 +0,0 @@ -import org.gradle.plugins.ide.eclipse.model.AccessRule -apply plugin: 'eclipse' - -eclipse.project.file.withXml { provider -> - ignoreDerivedResources(provider.asNode()) -} - -def ignoreDerivedResources(projectDescription, directories = ["build", ".gradle", ".git", ".settings"]) { - def count = directories.count { file(it).exists() } - if (count > 0) { - def filter = projectDescription - .appendNode("filteredResources") - .appendNode("filter") - filter.appendNode("id", System.currentTimeMillis().toString().trim()) - filter.appendNode("type", "26") - filter.appendNode("name") - def matcher = filter.appendNode("matcher") - matcher.appendNode("id", "org.eclipse.ui.ide.orFilterMatcher") - def arguments = matcher.appendNode("arguments") - directories.each { - def dirMatcher = arguments.appendNode("matcher") - dirMatcher.appendNode("id", "org.eclipse.ui.ide.multiFilter") - dirMatcher.appendNode("arguments", "1.0-projectRelativePath-matches-false-false-${it}") - } - } -} - -eclipse.classpath.file{ - whenMerged { classpath -> - classpath.getEntries().each{ entry -> - if(entry.getKind().equals("con")){ //there should only ever be one container, the JDK - //println("DEBUG: entry is: " + entry) - AccessRule javafxRule = new AccessRule("accessible", "javafx/**") - entry.getAccessRules().add(javafxRule) - } - } - } -} \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index f6b961fd5..000000000 Binary files a/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 2d80b69a7..000000000 --- a/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew deleted file mode 100755 index cccdd3d51..000000000 --- a/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat deleted file mode 100644 index e95643d6a..000000000 --- a/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/main.py b/main.py new file mode 100644 index 000000000..d56729d3c --- /dev/null +++ b/main.py @@ -0,0 +1,25 @@ +# +# Copyright (C) 2026 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import xml.etree.ElementTree as ET + +def define_env(env): + tree = ET.parse("pom.xml") + root = tree.getroot() + ns = {"m": "http://maven.apache.org/POM/4.0.0"} + version = root.findtext("m:version", namespaces=ns) or "" + + env.variables["project_version"] = version diff --git a/maven-central.gradle b/maven-central.gradle deleted file mode 100644 index b2f4601aa..000000000 --- a/maven-central.gradle +++ /dev/null @@ -1,84 +0,0 @@ -//This build file is for all the configuration specifics of building and deploying to maven-central -apply plugin: 'maven' -apply plugin: 'signing' - -group = 'gov.loc' - -//javadocs and sources required for uploading to maven central -/*task javadocJar(type: Jar) { - group "Build" - description "Create the jar that contains all the class documentation (javadoc)." - classifier = 'javadoc' - from javadoc -} - -task sourcesJar(type: Jar) { - group "Build" - description "Create the jar that contains all the .class files." - classifier = 'sources' - from sourceSets.main.allSource -}*/ - -artifacts { - archives javadocJar - archives sourcesJar - archives jar -} - -//artifacts need to be signed in order to upload them to maven central -signing { -// required { !version.endsWith("SNAPSHOT") } //only sign an actual release - sign configurations.archives -} - -uploadArchives { - repositories { - mavenDeployer { - if(!project.hasProperty("ossrhUsername")){ //so travis CI doesn't break... - project.ext.ossrhUsername = "foo" - project.ext.ossrhPassword = "foo" - } - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - - repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - - snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - - pom.whenConfigured { pom -> - pom.dependencies = pom.dependencies.findAll { dep -> dep.scope != 'test' } // removes the test scoped ones - } - pom.project { - name 'bagit-java' - packaging 'jar' - // optionally artifactId can be defined here - description 'The BAGIT LIBRARY is a software library intended to support the creation, manipulation, and validation of bags. Its current version is 0.97. It is version aware with the earliest supported version being 0.93.' - url 'https://github.com/LibraryOfCongress/bagit-java' - - scm { - connection 'scm:git:https://github.com/LibraryOfCongress/bagit-java' - developerConnection 'scm:git:ssh://github.com/LibraryOfCongress/bagit-java' - url 'https://github.com/LibraryOfCongress/bagit-java' - } - - licenses { - license { - name 'No Copyright' - url 'https://github.com/LibraryOfCongress/bagit-java/blob/master/LICENSE.txt' - } - } - - developers { - developer { - id 'johnscancella' - name 'John Scancella' - email 'jsca@loc.gov' - } - } - } - } - } -} diff --git a/message-bundle.gradle b/message-bundle.gradle deleted file mode 100644 index 2c91cf822..000000000 --- a/message-bundle.gradle +++ /dev/null @@ -1,56 +0,0 @@ -//this build file is responsible for all the tasks related to internationalization messages -import java.util.Map.Entry; - -task checkMessageBundle(){ - description "Checks the message bundles(which are used for language translation) that all entries are used, " + - "and that there are no duplicates." - group "Verification" - inputs.files(fileTree(dir: "src/main/resources", include: "**/MessageBundle*.properties")) - outputs.dir("$buildDir/checkMessageBundle") //hack: define a output dir so gradle will check if up-to-date - - doLast{ - inputs.getFiles().each{File file -> - Set messageKeys = new HashSet<>() - file.eachLine {String line -> - if(line && !line.trim().startsWith('#')){ - String[] keyValue = checkMessageBundleLineIsCorrectlyFormatted(line, file) - - if(messageKeys.contains(keyValue[0])){ - throw new GradleException("Internationalization message bundle contains duplicate key [${keyValue[0]}]!") - } - messageKeys.add(keyValue[0]) - } - } - checkAllMessageKeysAreUsed(messageKeys) - } - } -} -check.dependsOn checkMessageBundle - -String[] checkMessageBundleLineIsCorrectlyFormatted(String line, File file){ - String[] keyValue = line.split("=", 2) - - if(keyValue.size() != 2 || keyValue[1].isEmpty()){ - throw new GradleException("Line [${line}] in file [${file}] is not a valid entry for the internationalization message bundle!") - } - - return keyValue -} - -void checkAllMessageKeysAreUsed(Set messageKeys){ - sourceSets.main.allJava.each { File file -> - file.eachLine{ String line -> - for(String key : messageKeys.clone()){ - if(line.contains(key)){ - messageKeys.remove(key) - } - } - } - } - - if(messageKeys.size() > 0){ - messageKeys.each{String key -> - throw new GradleException("[${key}] is listed in the internationalization message bundle but never actually used!") - } - } -} diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..eb8081afc --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,52 @@ +# +# Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +site_name: dans-bagit-lib +theme: + name: readthedocs + +repo_name: DANS-KNAW/dans-bagit-lib +repo_url: https://github.com/DANS-KNAW/dans-bagit-lib + +nav: + - Manual: + - Description: index.md + - Getting started: getting-started.md + - JavaDocs: api.md + - Context: context.md + - Development: + - Overview: dev.md + +extra: + deposit_directory: https://dans-knaw.github.io/dans-datastation-architecture/deposit-directory/ + dans_bagit_profile: https://dans-knaw.github.io/dans-bagit-profile/versions/1.0.0/ + +plugins: + - markdownextradata + - search + - macros + +markdown_extensions: + - attr_list + - admonition + - codehilite: + guess_lang: False + use_pygments: False + - def_list + - footnotes + - meta + - toc: + permalink: true diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000..4986016d5 --- /dev/null +++ b/pom.xml @@ -0,0 +1,255 @@ + + + + 4.0.0 + + + nl.datastations + dans-core-systems-parent + 0.2.0 + + + dans-bagit-lib + 2.0.0-SNAPSHOT + + DANS BagIt Library + The BAGIT LIBRARY is a software library intended to support the creation, manipulation, and validation of bags. Its current version is 0.97. It is version aware with the earliest supported version being 0.93. + https://github.com/DANS-KNAW/dans-bagit-lib + 2023 + + + scm:git:https://github.com/DANS-KNAW/${project.artifactId} + scm:git:ssh://github.com/DANS-KNAW/${project.artifactId} + HEAD + + + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + org.apiguardian + apiguardian-api + + + org.bouncycastle + bcprov-jdk18on + + + org.kamranzafar + jtar + + + + + ch.qos.logback + logback-classic + + + org.slf4j + slf4j-api + + + + + org.junit.jupiter + junit-jupiter-api + test + + + + + + com.mycila + license-maven-plugin + + + bagit-conformance-suite/** + site/** + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-integration-test-source + + add-test-source + + generate-test-sources + + + src/integration/java + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + private + true + + + + + + org.apache.maven.plugins + maven-site-plugin + 3.21.0 + + + + + org.jacoco + jacoco-maven-plugin + + + gov/loc/repository/bagit/domain/** + gov/loc/repository/bagit/annotations/** + gov/loc/repository/bagit/exceptions/** + + + + + default-prepare-agent + + prepare-agent + + + + default-report + + report + + + + default-check + + check + + + + + BUNDLE + + + COMPLEXITY + COVEREDRATIO + 0.60 + + + + + + + + + + + org.apache.maven.plugins + maven-release-plugin + + + + + + + + + + org.apache.maven.plugins + maven-jxr-plugin + 3.6.0 + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.28.0 + + + src/main/resources/ruleset.xml + + + + + + + org.jacoco + jacoco-maven-plugin + + + gov/loc/repository/bagit/domain/** + gov/loc/repository/bagit/annotations/** + gov/loc/repository/bagit/exceptions/** + + + + + + + report + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + 4.10.4.0 + + + + org.apache.maven.plugins + maven-javadoc-plugin + + private + true + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 3.9.0 + + + + + diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index 75976b0fa..000000000 --- a/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = 'bagit' diff --git a/src/integration/java/gov/loc/repository/bagit/BagitSuiteComplanceTest.java b/src/integration/java/gov/loc/repository/bagit/BagitSuiteComplanceTest.java deleted file mode 100644 index 24cd32220..000000000 --- a/src/integration/java/gov/loc/repository/bagit/BagitSuiteComplanceTest.java +++ /dev/null @@ -1,194 +0,0 @@ -package gov.loc.repository.bagit; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicLong; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import gov.loc.repository.bagit.conformance.BagLinter; -import gov.loc.repository.bagit.conformance.BagitWarning; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.CorruptChecksumException; -import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.MissingBagitFileException; -import gov.loc.repository.bagit.exceptions.MissingPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.MissingPayloadManifestException; -import gov.loc.repository.bagit.exceptions.UnparsableVersionException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.exceptions.VerificationException; -import gov.loc.repository.bagit.reader.BagReader; -import gov.loc.repository.bagit.verify.BagVerifier; -import gov.loc.repository.bagit.writer.BagWriter; - -/** - * This class assumes that the compliance test suite repo has been cloned and is available locally - */ -public class BagitSuiteComplanceTest extends TempFolderTest { - private static final Logger logger = LoggerFactory.getLogger(BagitSuiteComplanceTest.class); - - private static final Path complianceRepoRootDir = Paths.get("bagit-conformance-suite"); - private static final BagTestCaseVistor visitor = new BagTestCaseVistor(); - private static final BagReader reader = new BagReader(); - private static final BagVerifier verifier = new BagVerifier(); - - @BeforeAll - public static void setupOnce() throws IOException{ - if(!Files.exists(complianceRepoRootDir)){ - throw new IOException("bagit-conformance-suite git repo was not found, did you clone it?"); - } - Files.walkFileTree(complianceRepoRootDir, visitor); - } - - @Test - public void testValidBags() throws Exception{ - Bag bag; - - for(final Path bagDir : visitor.getValidTestCases()){ - bag = reader.read(bagDir); - verifier.isValid(bag, true); - } - } - - @Test - public void testInvalidBags(){ - int errorCount = 0; - Bag bag; - ConcurrentMap, AtomicLong> map = new ConcurrentHashMap<>(); - - for(Path invalidBagDir : visitor.getInvalidTestCases()){ - try{ - bag = reader.read(invalidBagDir); - verifier.isValid(bag, true); - System.err.println(bag.getRootDir() + " should have failed but didn't!"); - }catch(InvalidBagitFileFormatException | IOException | UnparsableVersionException | - MissingPayloadManifestException | MissingBagitFileException | MissingPayloadDirectoryException | - FileNotInPayloadDirectoryException | InterruptedException | MaliciousPathException | - CorruptChecksumException | VerificationException | UnsupportedAlgorithmException e){ - - logger.info("Found invalid os specific bag with message: {}", e.getMessage()); - map.putIfAbsent(e.getClass(), new AtomicLong(0)); - map.get(e.getClass()).incrementAndGet(); - errorCount++; - } - } - - Assertions.assertEquals(visitor.getInvalidTestCases().size(), errorCount, "every test case should throw an error"); - logger.debug("Count of all errors found in generic invalid cases: {}", map); - } - - @Test - public void testInvalidOperatingSystemSpecificBags(){ - int errorCount = 0; - Bag bag; - List osSpecificInvalidPaths = visitor.getLinuxOnlyTestCases(); - ConcurrentMap, AtomicLong> map = new ConcurrentHashMap<>(); - - if(TestUtils.isExecutingOnWindows()){ - osSpecificInvalidPaths = visitor.getWindowsOnlyTestCases(); - } - - for(Path invalidBagDir : osSpecificInvalidPaths){ - try{ - bag = reader.read(invalidBagDir); - verifier.isValid(bag, true); - }catch(InvalidBagitFileFormatException | IOException | UnparsableVersionException | - MissingPayloadManifestException | MissingBagitFileException | MissingPayloadDirectoryException | - FileNotInPayloadDirectoryException | InterruptedException | MaliciousPathException | - CorruptChecksumException | VerificationException | UnsupportedAlgorithmException e){ - - logger.info("Found invalid os specific bag with message: {}", e.getMessage()); - map.putIfAbsent(e.getClass(), new AtomicLong(0)); - map.get(e.getClass()).incrementAndGet(); - errorCount++; - } - } - - Assertions.assertEquals(osSpecificInvalidPaths.size(), errorCount, "every test case should throw an error"); - logger.debug("Count of all errors found in os specific invalid cases: {}", map); - } - - @Test - public void testWarnings() throws Exception{ - Set warnings; - - for(Path bagDir : visitor.getWarningTestCases()){ - warnings = BagLinter.lintBag(bagDir); - Assertions.assertTrue(warnings.size() > 0); - } - } - - @Test - public void testReadWriteProducesSameBag() throws Exception{ - Bag bag; - Path newBagDir; - - for(final Path bagDir : visitor.getValidTestCases()){ - newBagDir = folder.resolve("readWriteProducesSameBag"); - bag = reader.read(bagDir); - BagWriter.write(bag, newBagDir); - - testTagFileContents(bag, newBagDir); - - testBagsStructureAreEqual(bagDir, newBagDir); - } - } - - private void testTagFileContents(final Bag originalBag, final Path newBagDir) throws IOException{ - Path original = originalBag.getRootDir().resolve("bagit.txt"); - Path newFile = newBagDir.resolve("bagit.txt"); - Assertions.assertTrue(compareFileContents(original, newFile, StandardCharsets.UTF_8), "bagit.txt files differ"); - - if(originalBag.getVersion().isSameOrOlder(new Version(0, 95))){ - original = originalBag.getRootDir().resolve("package-info.txt"); - newFile = newBagDir.resolve("package-info.txt"); - Assertions.assertTrue(compareFileContents(original, newFile, originalBag.getFileEncoding()), - original + " differs from " + newFile); - } - else{ - if(Files.exists(originalBag.getRootDir().resolve("bag-info.txt"))){ - original = originalBag.getRootDir().resolve("bag-info.txt"); - newFile = newBagDir.resolve("bag-info.txt"); - Assertions.assertTrue(compareFileContents(original,newFile, originalBag.getFileEncoding()), - original + " differs from " + newFile); - } - } - - } - - //return true if the content is the same disregarding line endings - private final boolean compareFileContents(final Path file1, final Path file2, final Charset encoding) throws IOException { - List lines1 = Files.readAllLines(file1, encoding); - List lines2 = Files.readAllLines(file2, encoding); - - List strippedLines1 = new ArrayList<>(lines1.size()); - List strippedLines2 = new ArrayList<>(lines2.size()); - - for(int index=0; index, AtomicLong> map = new ConcurrentHashMap<>(); + + for (Path invalidBagDir : visitor.getInvalidTestCases()) { + try { + bag = reader.read(invalidBagDir); + verifier.isValid(bag, true); + System.err.println(bag.getRootDir() + " should have failed but didn't!"); + } + catch (InvalidBagitFileFormatException | IOException | UnparsableVersionException | + MissingPayloadManifestException | MissingBagitFileException | MissingPayloadDirectoryException | + FileNotInPayloadDirectoryException | InterruptedException | MaliciousPathException | + CorruptChecksumException | VerificationException | UnsupportedAlgorithmException e) { + + logger.info("Found invalid os specific bag with message: {}", e.getMessage()); + map.putIfAbsent(e.getClass(), new AtomicLong(0)); + map.get(e.getClass()).incrementAndGet(); + errorCount++; + } + } + + Assertions.assertEquals(visitor.getInvalidTestCases().size(), errorCount, "every test case should throw an error"); + logger.debug("Count of all errors found in generic invalid cases: {}", map); + } + + @Test + public void testInvalidOperatingSystemSpecificBags() { + int errorCount = 0; + Bag bag; + List osSpecificInvalidPaths = visitor.getLinuxOnlyTestCases(); + ConcurrentMap, AtomicLong> map = new ConcurrentHashMap<>(); + + if (TestUtils.isExecutingOnWindows()) { + osSpecificInvalidPaths = visitor.getWindowsOnlyTestCases(); + } + + for (Path invalidBagDir : osSpecificInvalidPaths) { + try { + bag = reader.read(invalidBagDir); + verifier.isValid(bag, true); + } + catch (InvalidBagitFileFormatException | IOException | UnparsableVersionException | + MissingPayloadManifestException | MissingBagitFileException | MissingPayloadDirectoryException | + FileNotInPayloadDirectoryException | InterruptedException | MaliciousPathException | + CorruptChecksumException | VerificationException | UnsupportedAlgorithmException e) { + + logger.info("Found invalid os specific bag with message: {}", e.getMessage()); + map.putIfAbsent(e.getClass(), new AtomicLong(0)); + map.get(e.getClass()).incrementAndGet(); + errorCount++; + } + } + + Assertions.assertEquals(osSpecificInvalidPaths.size(), errorCount, "every test case should throw an error"); + logger.debug("Count of all errors found in os specific invalid cases: {}", map); + } + + @Test + public void testWarnings() throws Exception { + Set warnings; + + for (Path bagDir : visitor.getWarningTestCases()) { + warnings = BagLinter.lintBag(bagDir); + Assertions.assertFalse(warnings.isEmpty()); + } + } + + @Test + public void testReadWriteProducesSameBag() throws Exception { + Bag bag; + Path newBagDir; + + for (final Path bagDir : visitor.getValidTestCases()) { + newBagDir = folder.resolve("readWriteProducesSameBag"); + bag = reader.read(bagDir); + BagWriter.write(bag, newBagDir); + + testTagFileContents(bag, newBagDir); + + testBagsStructureAreEqual(bagDir, newBagDir); + delete(newBagDir); + } + } + + private void testTagFileContents(final Bag originalBag, final Path newBagDir) throws IOException { + Path original = originalBag.getRootDir().resolve("bagit.txt"); + Path newFile = newBagDir.resolve("bagit.txt"); + Assertions.assertTrue(compareFileContents(original, newFile, StandardCharsets.UTF_8), "bagit.txt files differ"); + + if (originalBag.getVersion().isSameOrOlder(new Version(0, 95))) { + original = originalBag.getRootDir().resolve("package-info.txt"); + newFile = newBagDir.resolve("package-info.txt"); + Assertions.assertTrue(compareFileContents(original, newFile, originalBag.getFileEncoding()), + original + " differs from " + newFile); + } + else { + if (Files.exists(originalBag.getRootDir().resolve("bag-info.txt"))) { + original = originalBag.getRootDir().resolve("bag-info.txt"); + newFile = newBagDir.resolve("bag-info.txt"); + Assertions.assertTrue(compareFileContents(original, newFile, originalBag.getFileEncoding()), + original + " differs from " + newFile); + } + } + + } + + //return true if the content is the same disregarding line endings and indentation + private boolean compareFileContents(final Path file1, final Path file2, final Charset encoding) throws IOException { + List lines1 = Files.readAllLines(file1, encoding); + List lines2 = Files.readAllLines(file2, encoding); + + String s1 = String.join("", lines1).replaceAll("\\r|\\n| |\t", "").replaceAll("Payload-Oxum:[0-9.]+", ""); + String s2 = String.join("", lines2).replaceAll("\\r|\\n| |\t", "").replaceAll("Payload-Oxum:[0-9.]+", ""); + + return s1.equals(s2); + } + + private void testBagsStructureAreEqual(Path originalBag, Path newBag) throws IOException { + Files.walkFileTree(originalBag, new FileExistsVistor(originalBag, newBag)); + } +} diff --git a/src/integration/java/gov/loc/repository/bagit/FileExistsVistor.java b/src/integration/java/nl/knaw/dans/bagit/FileExistsVistor.java similarity index 62% rename from src/integration/java/gov/loc/repository/bagit/FileExistsVistor.java rename to src/integration/java/nl/knaw/dans/bagit/FileExistsVistor.java index 0a3b4db3e..3452cfff8 100644 --- a/src/integration/java/gov/loc/repository/bagit/FileExistsVistor.java +++ b/src/integration/java/nl/knaw/dans/bagit/FileExistsVistor.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit; import java.io.IOException; import java.nio.file.FileVisitResult; diff --git a/src/integration/java/gov/loc/repository/bagit/ReaderWriterVerifierIntegrationTest.java b/src/integration/java/nl/knaw/dans/bagit/ReaderWriterVerifierIntegrationTest.java similarity index 80% rename from src/integration/java/gov/loc/repository/bagit/ReaderWriterVerifierIntegrationTest.java rename to src/integration/java/nl/knaw/dans/bagit/ReaderWriterVerifierIntegrationTest.java index 2aa96dfcf..3a2fcdd0e 100644 --- a/src/integration/java/gov/loc/repository/bagit/ReaderWriterVerifierIntegrationTest.java +++ b/src/integration/java/nl/knaw/dans/bagit/ReaderWriterVerifierIntegrationTest.java @@ -1,16 +1,31 @@ -package gov.loc.repository.bagit; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import nl.knaw.dans.bagit.domain.Bag; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.reader.BagReader; -import gov.loc.repository.bagit.verify.BagVerifier; -import gov.loc.repository.bagit.writer.BagWriter; +import nl.knaw.dans.bagit.reader.BagReader; +import nl.knaw.dans.bagit.verify.BagVerifier; +import nl.knaw.dans.bagit.writer.BagWriter; public class ReaderWriterVerifierIntegrationTest extends TempFolderTest { diff --git a/src/main/java/gov/loc/repository/bagit/conformance/profile/Serialization.java b/src/main/java/gov/loc/repository/bagit/conformance/profile/Serialization.java deleted file mode 100644 index 3ecb73379..000000000 --- a/src/main/java/gov/loc/repository/bagit/conformance/profile/Serialization.java +++ /dev/null @@ -1,10 +0,0 @@ -package gov.loc.repository.bagit.conformance.profile; - -/** - * The type of serialization required by a {@link BagitProfile} - */ -public enum Serialization { - forbidden, - required, - optional; -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/CorruptChecksumException.java b/src/main/java/gov/loc/repository/bagit/exceptions/CorruptChecksumException.java deleted file mode 100644 index 1e947bdcf..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/CorruptChecksumException.java +++ /dev/null @@ -1,16 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -import java.nio.file.Path; - -import org.slf4j.helpers.MessageFormatter; - -/** - * Class to represent an error when the calculated checksum is different than the manifest specified checksum. - */ -public class CorruptChecksumException extends Exception { - private static final long serialVersionUID = 1L; - - public CorruptChecksumException(final String message, final Path path, final String algorithm, final String hash, final String computedHash){ - super(MessageFormatter.arrayFormat(message, new Object[]{path, algorithm, hash, computedHash}).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/FileNotInManifestException.java b/src/main/java/gov/loc/repository/bagit/exceptions/FileNotInManifestException.java deleted file mode 100644 index 7447a220c..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/FileNotInManifestException.java +++ /dev/null @@ -1,15 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -import java.io.IOException; - -/** - * Class to represent an error when a file is found in the payload directory but not in any manifest. - * Opposite to {@link FileNotInPayloadDirectoryException} - */ -public class FileNotInManifestException extends IOException { - private static final long serialVersionUID = 1L; - - public FileNotInManifestException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/FileNotInPayloadDirectoryException.java b/src/main/java/gov/loc/repository/bagit/exceptions/FileNotInPayloadDirectoryException.java deleted file mode 100644 index 870f9a658..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/FileNotInPayloadDirectoryException.java +++ /dev/null @@ -1,13 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * Class to represent an error when a file is not in the payload directory but is listed in a manifest. - * Opposite to {@link FileNotInManifestException} - */ -public class FileNotInPayloadDirectoryException extends Exception { - private static final long serialVersionUID = 1L; - - public FileNotInPayloadDirectoryException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/InvalidBagMetadataException.java b/src/main/java/gov/loc/repository/bagit/exceptions/InvalidBagMetadataException.java deleted file mode 100644 index 8ae449301..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/InvalidBagMetadataException.java +++ /dev/null @@ -1,17 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * Class to represent an error when the bag metadata file does not conform to the bagit spec, - * namely:
- * <KEY>:<VALUE> - *
or - * 
<KEY>:<VALUE>
- *    <VALUE CONTINUED>
- */ -public class InvalidBagMetadataException extends InvalidBagitFileFormatException { - private static final long serialVersionUID = 1L; - - public InvalidBagMetadataException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/InvalidBagitFileFormatException.java b/src/main/java/gov/loc/repository/bagit/exceptions/InvalidBagitFileFormatException.java deleted file mode 100644 index 8293d9d78..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/InvalidBagitFileFormatException.java +++ /dev/null @@ -1,16 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * Class to represent an error when a specific bag file does not conform to its bagit specfication format - */ -public class InvalidBagitFileFormatException extends Exception { - private static final long serialVersionUID = 1L; - - public InvalidBagitFileFormatException(final String message){ - super(message); - } - - public InvalidBagitFileFormatException(final String message, final Exception e){ - super(message, e); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/InvalidPayloadOxumException.java b/src/main/java/gov/loc/repository/bagit/exceptions/InvalidPayloadOxumException.java deleted file mode 100644 index e48f17cea..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/InvalidPayloadOxumException.java +++ /dev/null @@ -1,13 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * Class to represent an error when the calculated total bytes or number of files for - * the payload-oxum is different than the supplied values. - */ -public class InvalidPayloadOxumException extends Exception { - private static final long serialVersionUID = 1L; - - public InvalidPayloadOxumException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/MaliciousPathException.java b/src/main/java/gov/loc/repository/bagit/exceptions/MaliciousPathException.java deleted file mode 100644 index 48fd66a15..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/MaliciousPathException.java +++ /dev/null @@ -1,13 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * Class to represent an error when the path in a manifest or fetch file has been crafted to point to a file or - * directory outside the bag. Most likely to try and overwrite an important system file. - */ -public class MaliciousPathException extends Exception { - private static final long serialVersionUID = 1L; - - public MaliciousPathException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/MissingBagitFileException.java b/src/main/java/gov/loc/repository/bagit/exceptions/MissingBagitFileException.java deleted file mode 100644 index c7c45c7d0..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/MissingBagitFileException.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * The bagit.txt file is a required file. This class represents the error if that file is not present. - */ -public class MissingBagitFileException extends Exception { - private static final long serialVersionUID = 1L; - - public MissingBagitFileException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadDirectoryException.java b/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadDirectoryException.java deleted file mode 100644 index 766b05aa7..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadDirectoryException.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * The payload directory is a required file. This class represents the error if it is not found. - */ -public class MissingPayloadDirectoryException extends Exception { - private static final long serialVersionUID = 1L; - - public MissingPayloadDirectoryException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadManifestException.java b/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadManifestException.java deleted file mode 100644 index 7cc9b9d6a..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadManifestException.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * A bagit bag needs at least one payload manifest. This class represents the error if at least one payload manifest isn't found. - */ -public class MissingPayloadManifestException extends Exception { - private static final long serialVersionUID = 1L; - - public MissingPayloadManifestException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/PayloadOxumDoesNotExistException.java b/src/main/java/gov/loc/repository/bagit/exceptions/PayloadOxumDoesNotExistException.java deleted file mode 100644 index 476f532c6..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/PayloadOxumDoesNotExistException.java +++ /dev/null @@ -1,15 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -import gov.loc.repository.bagit.domain.Bag; - -/** - * The {@link Bag} object should contain the Payload-Oxum metatdata key value pair, - * this class represents the error when trying to calculate the payload-oxum and it doesn't exist on the bag object. - */ -public class PayloadOxumDoesNotExistException extends RuntimeException { - private static final long serialVersionUID = 1L; - - public PayloadOxumDoesNotExistException(final String message){ - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/UnparsableVersionException.java b/src/main/java/gov/loc/repository/bagit/exceptions/UnparsableVersionException.java deleted file mode 100644 index 2394c9da7..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/UnparsableVersionException.java +++ /dev/null @@ -1,14 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -import org.slf4j.helpers.MessageFormatter; - -/** - * If the version string in the bagit.txt file was not in the form <MAJOR>.<MINOR> - */ -public class UnparsableVersionException extends Exception { - private static final long serialVersionUID = 1L; - - public UnparsableVersionException(final String message, final String version){ - super(MessageFormatter.format(message, version).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/UnsupportedAlgorithmException.java b/src/main/java/gov/loc/repository/bagit/exceptions/UnsupportedAlgorithmException.java deleted file mode 100644 index 5b7ee7b83..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/UnsupportedAlgorithmException.java +++ /dev/null @@ -1,16 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -import java.security.MessageDigest; - -import org.slf4j.helpers.MessageFormatter; - -/** - * When the bag uses an checksum algorithm that is not supported by {@link MessageDigest}. - */ -public class UnsupportedAlgorithmException extends Exception { - private static final long serialVersionUID = 1L; - - public UnsupportedAlgorithmException(final String message, final String bagitAlgorithmName, final Throwable cause) { - super(MessageFormatter.format(message, bagitAlgorithmName).getMessage(), cause); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/VerificationException.java b/src/main/java/gov/loc/repository/bagit/exceptions/VerificationException.java deleted file mode 100644 index 3f1f44906..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/VerificationException.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.loc.repository.bagit.exceptions; - -/** - * Class to represent an generic exception that happened during verification. - */ -public class VerificationException extends Exception { - private static final long serialVersionUID = 1L; - - public VerificationException(final Exception exception){ - super(exception); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/BagitVersionIsNotAcceptableException.java b/src/main/java/gov/loc/repository/bagit/exceptions/conformance/BagitVersionIsNotAcceptableException.java deleted file mode 100644 index 3a848c66e..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/BagitVersionIsNotAcceptableException.java +++ /dev/null @@ -1,18 +0,0 @@ -package gov.loc.repository.bagit.exceptions.conformance; - -import java.util.List; - -import org.slf4j.helpers.MessageFormatter; - -import gov.loc.repository.bagit.domain.Version; - -/** - * Class to represent when the bag's version is not in the acceptable list of versions - */ -public class BagitVersionIsNotAcceptableException extends Exception { -private static final long serialVersionUID = 1L; - - public BagitVersionIsNotAcceptableException(final String message, final Version version, final List acceptableVersions) { - super(MessageFormatter.format(message, version, acceptableVersions).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/FetchFileNotAllowedException.java b/src/main/java/gov/loc/repository/bagit/exceptions/conformance/FetchFileNotAllowedException.java deleted file mode 100644 index 734a2c0c0..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/FetchFileNotAllowedException.java +++ /dev/null @@ -1,16 +0,0 @@ -package gov.loc.repository.bagit.exceptions.conformance; - -import java.nio.file.Path; - -import org.slf4j.helpers.MessageFormatter; - -/** - * Class to represent when a fetch file is found in a bag but is not allowed according to the bagit profile - */ -public class FetchFileNotAllowedException extends Exception { -private static final long serialVersionUID = 1L; - - public FetchFileNotAllowedException(final String message, final Path rootDir) { - super(MessageFormatter.format(message, rootDir).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/MetatdataValueIsNotAcceptableException.java b/src/main/java/gov/loc/repository/bagit/exceptions/conformance/MetatdataValueIsNotAcceptableException.java deleted file mode 100644 index 7989db7c6..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/MetatdataValueIsNotAcceptableException.java +++ /dev/null @@ -1,16 +0,0 @@ -package gov.loc.repository.bagit.exceptions.conformance; - -import java.util.List; - -import org.slf4j.helpers.MessageFormatter; - -/** - * Class to represent when a metadata's value is not in the acceptable list of values - */ -public class MetatdataValueIsNotAcceptableException extends Exception { -private static final long serialVersionUID = 1L; - - public MetatdataValueIsNotAcceptableException(final String message, final String metadataKey, final List acceptableValues, final String actualValue) { - super(MessageFormatter.arrayFormat(message, new Object[]{metadataKey, acceptableValues, actualValue}).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/MetatdataValueIsNotRepeatableException.java b/src/main/java/gov/loc/repository/bagit/exceptions/conformance/MetatdataValueIsNotRepeatableException.java deleted file mode 100644 index f9c15ae75..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/MetatdataValueIsNotRepeatableException.java +++ /dev/null @@ -1,14 +0,0 @@ -package gov.loc.repository.bagit.exceptions.conformance; - -import org.slf4j.helpers.MessageFormatter; - -/** - * Class to represent when a metadata's value is not to be repeated - */ -public class MetatdataValueIsNotRepeatableException extends Exception { -private static final long serialVersionUID = 1L; - - public MetatdataValueIsNotRepeatableException(final String message, final String metadataKey) { - super(MessageFormatter.format(message, metadataKey).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredManifestNotPresentException.java b/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredManifestNotPresentException.java deleted file mode 100644 index e628bc0ae..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredManifestNotPresentException.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.loc.repository.bagit.exceptions.conformance; - -/** - * Class to represent when a specific manifest type is not found, such as md5, sha1, etc (payload or tag) - */ -public class RequiredManifestNotPresentException extends Exception { -private static final long serialVersionUID = 1L; - - public RequiredManifestNotPresentException(final String message) { - super(message); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredMetadataFieldNotPresentException.java b/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredMetadataFieldNotPresentException.java deleted file mode 100644 index 8af60d064..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredMetadataFieldNotPresentException.java +++ /dev/null @@ -1,14 +0,0 @@ -package gov.loc.repository.bagit.exceptions.conformance; - -import org.slf4j.helpers.MessageFormatter; - -/** - * Class to represent when a specific metadata field is not found - */ -public class RequiredMetadataFieldNotPresentException extends Exception { -private static final long serialVersionUID = 1L; - - public RequiredMetadataFieldNotPresentException(final String message, final String bagInfoEntryRequirementKey) { - super(MessageFormatter.format(message, bagInfoEntryRequirementKey).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredTagFileNotPresentException.java b/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredTagFileNotPresentException.java deleted file mode 100644 index 5bbc93d53..000000000 --- a/src/main/java/gov/loc/repository/bagit/exceptions/conformance/RequiredTagFileNotPresentException.java +++ /dev/null @@ -1,14 +0,0 @@ -package gov.loc.repository.bagit.exceptions.conformance; - -import org.slf4j.helpers.MessageFormatter; - -/** - * Class to represent when a specific tag file is not found - */ -public class RequiredTagFileNotPresentException extends Exception { -private static final long serialVersionUID = 1L; - - public RequiredTagFileNotPresentException(final String message, final String requiredTagFilePath) { - super(MessageFormatter.format(message, requiredTagFilePath).getMessage()); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/hash/BagitAlgorithmNameToSupportedAlgorithmMapping.java b/src/main/java/gov/loc/repository/bagit/hash/BagitAlgorithmNameToSupportedAlgorithmMapping.java deleted file mode 100644 index 96dc0a21d..000000000 --- a/src/main/java/gov/loc/repository/bagit/hash/BagitAlgorithmNameToSupportedAlgorithmMapping.java +++ /dev/null @@ -1,10 +0,0 @@ -package gov.loc.repository.bagit.hash; - -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; - -/** - * Implement this interface if you need to be able to use other algorithms than the {@link StandardSupportedAlgorithms} - */ -public interface BagitAlgorithmNameToSupportedAlgorithmMapping { - SupportedAlgorithm getSupportedAlgorithm(String bagitAlgorithmName) throws UnsupportedAlgorithmException; -} diff --git a/src/main/java/gov/loc/repository/bagit/hash/Hasher.java b/src/main/java/gov/loc/repository/bagit/hash/Hasher.java deleted file mode 100644 index 55816ac52..000000000 --- a/src/main/java/gov/loc/repository/bagit/hash/Hasher.java +++ /dev/null @@ -1,115 +0,0 @@ -package gov.loc.repository.bagit.hash; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Formatter; -import java.util.HashMap; -import java.util.Map; -import java.util.ResourceBundle; -import java.util.Map.Entry; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import gov.loc.repository.bagit.domain.Manifest; - -/** - * Convenience class for generating a HEX formatted string of the checksum hash. - */ -public final class Hasher { - private static final Logger logger = LoggerFactory.getLogger(Hasher.class); - private static final int _64_KB = 1024 * 64; - private static final int CHUNK_SIZE = _64_KB; - private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); - - private Hasher(){ - //intentionally left empty - } - - /** - * Create a HEX formatted string checksum hash of the file - * - * @param path the {@link Path} (file) to hash - * @param messageDigest the {@link MessageDigest} object representing the hashing algorithm - * @return the hash as a hex formated string - * @throws IOException if there is a problem reading the file - */ - public static String hash(final Path path, final MessageDigest messageDigest) throws IOException { - updateMessageDigests(path, Arrays.asList(messageDigest)); - - return formatMessageDigest(messageDigest); - } - - /** - * Update the Manifests with the file's hash - * - * @param path the {@link Path} (file) to hash - * @param manifestToMessageDigestMap the map between {@link Manifest} and {@link MessageDigest} - * @throws IOException if there is a problem reading the file - */ - public static void hash(final Path path, final Map manifestToMessageDigestMap) throws IOException { - updateMessageDigests(path, manifestToMessageDigestMap.values()); - addMessageDigestHashToManifest(path, manifestToMessageDigestMap); - } - - static void updateMessageDigests(final Path path, final Collection messageDigests) throws IOException{ - try(final InputStream is = new BufferedInputStream(Files.newInputStream(path, StandardOpenOption.READ))){ - final byte[] buffer = new byte[CHUNK_SIZE]; - int read = is.read(buffer); - - while(read != -1) { - for(final MessageDigest messageDigest : messageDigests){ - messageDigest.update(buffer, 0, read); - } - read = is.read(buffer); - } - } - } - - private static void addMessageDigestHashToManifest(final Path path, final Map manifestToMessageDigestMap){ - for(final Entry entry : manifestToMessageDigestMap.entrySet()){ - final String hash = formatMessageDigest(entry.getValue()); - logger.debug(messages.getString("adding_checksum"), path, hash); - entry.getKey().getFileToChecksumMap().put(path, hash); - } - } - - //Convert the byte to hex format - private static String formatMessageDigest(final MessageDigest messageDigest){ - try(final Formatter formatter = new Formatter()){ - for (final byte b : messageDigest.digest()) { - formatter.format("%02x", b); - } - - return formatter.toString(); - } - } - - /** - * create a mapping between {@link Manifest} and {@link MessageDigest} for each each supplied {@link SupportedAlgorithm} - * - * @param algorithms the {@link SupportedAlgorithm} that you which to map to {@link MessageDigest} - * @return mapping between {@link Manifest} and {@link MessageDigest} - * @throws NoSuchAlgorithmException if {@link MessageDigest} doesn't support the algorithm - */ - @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") - public static Map createManifestToMessageDigestMap(final Collection algorithms) throws NoSuchAlgorithmException{ - final Map map = new HashMap<>(); - - for(final SupportedAlgorithm algorithm : algorithms){ - final MessageDigest messageDigest = MessageDigest.getInstance(algorithm.getMessageDigestName()); - final Manifest manifest = new Manifest(algorithm); - map.put(manifest, messageDigest); - } - - return map; - } -} diff --git a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java b/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java deleted file mode 100644 index 40818316c..000000000 --- a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java +++ /dev/null @@ -1,29 +0,0 @@ -package gov.loc.repository.bagit.hash; - -/** - * The standard algorithms that are supported "out of the box" in bagit - */ -public enum StandardSupportedAlgorithms implements SupportedAlgorithm{ - MD5("MD5"), - SHA1("SHA-1"), - SHA224("SHA-224"), - SHA256("SHA-256"), - SHA512("SHA-512"); - - private final String messageDigestName; - - private StandardSupportedAlgorithms(final String messageDigestName){ - this.messageDigestName = messageDigestName; - } - - @Override - public String getMessageDigestName() { - return messageDigestName; - } - - @SuppressWarnings({"PMD.UseLocaleWithCaseConversions"}) - @Override - public String getBagitName() { - return name().toLowerCase(); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/hash/SupportedAlgorithm.java b/src/main/java/gov/loc/repository/bagit/hash/SupportedAlgorithm.java deleted file mode 100644 index d5558ca63..000000000 --- a/src/main/java/gov/loc/repository/bagit/hash/SupportedAlgorithm.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.loc.repository.bagit.hash; - -import java.security.MessageDigest; - -/** - * Easy way to convert between bagit manifest spec and {@link MessageDigest}
- * See {@link StandardSupportedAlgorithms} for a list of defaults - */ -public interface SupportedAlgorithm { - String getMessageDigestName(); - String getBagitName(); -} diff --git a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java deleted file mode 100644 index 835e06beb..000000000 --- a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java +++ /dev/null @@ -1,222 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Map.Entry; -import java.util.ResourceBundle; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.exceptions.CorruptChecksumException; -import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.InvalidPayloadOxumException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.MissingBagitFileException; -import gov.loc.repository.bagit.exceptions.MissingPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.MissingPayloadManifestException; -import gov.loc.repository.bagit.exceptions.PayloadOxumDoesNotExistException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.exceptions.VerificationException; -import gov.loc.repository.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; - -/** - * Responsible for verifying if a bag is valid, complete - */ -public final class BagVerifier implements AutoCloseable{ - private static final Logger logger = LoggerFactory.getLogger(BagVerifier.class); - private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); - - private final PayloadVerifier manifestVerifier; - private final ExecutorService executor; - - /** - * Create a BagVerifier with a cached thread pool and a - * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} - */ - public BagVerifier(){ - this(Executors.newCachedThreadPool(), new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); - } - - /** - * Create a BagVerifier with a cached thread pool and a custom mapping - * - * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm - */ - public BagVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){ - this(Executors.newCachedThreadPool(), nameMapping); - } - - /** - * Create a BagVerifier with a custom thread pool and a - * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} - * - * @param executor the thread pool to use when doing work - */ - public BagVerifier(final ExecutorService executor){ - this(executor, new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); - } - - /** - * Create a BagVerifier with a custom thread pool and a custom mapping - * - * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm - * @param executor the thread pool to use when doing work - */ - public BagVerifier(final ExecutorService executor, final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){ - manifestVerifier = new PayloadVerifier(nameMapping, executor); - this.executor = executor; - } - - @Override - public void close() throws SecurityException{ - //shutdown the thread pool so the resource isn't leaked - executor.shutdown(); - manifestVerifier.close(); - } - - /** - * Determine if we can quickly verify by comparing the number of files and the total number of bytes expected - * - * @param bag the {@link Bag} object you wish to check - * @return true if the bag can be quickly verified - */ - public static boolean canQuickVerify(final Bag bag){ - return QuickVerifier.canQuickVerify(bag); - } - - /** - * Quickly verify by comparing the number of files and the total number of bytes expected - * - * @param bag the bag to verify by payload-oxum - * - * @throws IOException if there is an error reading a file - * @throws InvalidPayloadOxumException if either the total bytes or the number of files - * calculated for the payload directory of the bag is different than the supplied values - * @throws PayloadOxumDoesNotExistException if the bag does not contain a payload-oxum. - * To check, run {@link BagVerifier#canQuickVerify} - */ - public static void quicklyVerify(final Bag bag) throws IOException, InvalidPayloadOxumException{ - QuickVerifier.quicklyVerify(bag); - } - - /** - * See https://tools.ietf.org/html/draft-kunze-bagit-13#section-3
- * A bag is valid if the bag is complete and every checksum has been - * verified against the contents of its corresponding file. - * - * @param bag the {@link Bag} object to check - * @param ignoreHiddenFiles ignore hidden files unless explicitly listed in manifest(s) - * - * @throws CorruptChecksumException when the computed hash doesn't match given hash - * @throws IOException if there was an error with the file - * @throws MissingPayloadManifestException if there is not at least one payload manifest - * @throws MissingBagitFileException if there is no bagit.txt file - * @throws MissingPayloadDirectoryException if there is no /data directory - * @throws FileNotInPayloadDirectoryException if a manifest lists a file but it is not in the payload directory - * @throws InterruptedException if the threads are interrupted when checking if all files are listed in manifest(s) - * @throws MaliciousPathException if there is path that is referenced in the manifest that is outside the bag root directory - * @throws VerificationException some other exception happened during processing so capture it here. - * @throws UnsupportedAlgorithmException if the manifest uses a algorithm that isn't supported - * @throws InvalidBagitFileFormatException if the manifest is not formatted properly - */ - public void isValid(final Bag bag, final boolean ignoreHiddenFiles) throws IOException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ - logger.info(messages.getString("checking_bag_is_valid"), bag.getRootDir()); - isComplete(bag, ignoreHiddenFiles); - - logger.debug(messages.getString("checking_payload_checksums")); - for(final Manifest payloadManifest : bag.getPayLoadManifests()){ - checkHashes(payloadManifest); - } - - logger.debug(messages.getString("checking_tag_file_checksums")); - for(final Manifest tagManifest : bag.getTagManifests()){ - checkHashes(tagManifest); - } - } - - /* - * Check the supplied checksum hashes against the generated checksum hashes - */ - @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") - void checkHashes(final Manifest manifest) throws CorruptChecksumException, InterruptedException, VerificationException{ - final CountDownLatch latch = new CountDownLatch( manifest.getFileToChecksumMap().size()); - - //TODO maybe return all of these at some point... - final Collection exceptions = Collections.synchronizedCollection(new ArrayList<>()); - - for(final Entry entry : manifest.getFileToChecksumMap().entrySet()){ - executor.execute(new CheckManifestHashesTask(entry, manifest.getAlgorithm().getMessageDigestName(), latch, exceptions)); - } - - latch.await(); - - if(!exceptions.isEmpty()){ - final Exception e = exceptions.iterator().next(); - if(e instanceof CorruptChecksumException){ - logger.debug(messages.getString("checksums_not_matching_error"), exceptions.size()); - throw (CorruptChecksumException)e; - } - - throw new VerificationException(e); - } - } - - /** - * See https://tools.ietf.org/html/draft-kunze-bagit-13#section-3
- * A bag is complete if
- *
    - *
  • every element is present - *
  • every file in the payload manifest(s) are present - *
  • every file in the tag manifest(s) are present. Tag files not listed in a tag manifest may be present. - *
  • every file in the data directory must be listed in at least one payload manifest - *
  • each element must comply with the bagit spec - *
- * - * @param bag the {@link Bag} object to check - * @param ignoreHiddenFiles ignore hidden files unless explicitly listed in manifest(s) - * - * @throws IOException if there was an error with the file - * @throws MissingPayloadManifestException if there is not at least one payload manifest - * @throws MissingBagitFileException if there is no bagit.txt file - * @throws MissingPayloadDirectoryException if there is no /data directory - * @throws FileNotInPayloadDirectoryException if a manifest lists a file but it is not in the payload directory - * @throws InterruptedException if the threads are interrupted when checking if all files are listed in manifest(s) - * @throws MaliciousPathException if there is path that is referenced in the manifest that is outside the bag root directory - * @throws UnsupportedAlgorithmException if the manifest uses a algorithm that isn't supported - * @throws InvalidBagitFileFormatException if the manifest is not formatted properly - */ - public void isComplete(final Bag bag, final boolean ignoreHiddenFiles) throws - IOException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, - FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ - logger.info(messages.getString("checking_bag_is_complete"), bag.getRootDir()); - - MandatoryVerifier.checkFetchItemsExist(bag.getItemsToFetch(), bag.getRootDir()); - - MandatoryVerifier.checkBagitFileExists(bag.getRootDir(), bag.getVersion()); - - MandatoryVerifier.checkPayloadDirectoryExists(bag); - - MandatoryVerifier.checkIfAtLeastOnePayloadManifestsExist(bag.getRootDir(), bag.getVersion()); - - manifestVerifier.verifyPayload(bag, ignoreHiddenFiles); - } - - public ExecutorService getExecutor() { - return executor; - } - - public PayloadVerifier getManifestVerifier() { - return manifestVerifier; - } -} diff --git a/src/main/java/gov/loc/repository/bagit/verify/CheckManifestHashesTask.java b/src/main/java/gov/loc/repository/bagit/verify/CheckManifestHashesTask.java deleted file mode 100644 index b7153f0a2..000000000 --- a/src/main/java/gov/loc/repository/bagit/verify/CheckManifestHashesTask.java +++ /dev/null @@ -1,62 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Collection; -import java.util.Map.Entry; -import java.util.ResourceBundle; -import java.util.concurrent.CountDownLatch; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import gov.loc.repository.bagit.exceptions.CorruptChecksumException; -import gov.loc.repository.bagit.hash.Hasher; - -/** - * Checks a give file to make sure the given checksum hash matches the computed checksum hash. - * This is thread safe so you can call many at a time. - */ -@SuppressWarnings("PMD.DoNotUseThreads") -public class CheckManifestHashesTask implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(CheckManifestHashesTask.class); - private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); - - private transient final Entry entry; - private transient final CountDownLatch latch; - private transient final Collection exceptions; - private transient final String algorithm; - - public CheckManifestHashesTask(final Entry entry, final String algorithm, final CountDownLatch latch, final Collection exceptions) { - this.entry = entry; - this.algorithm = algorithm; - this.latch = latch; - this.exceptions = exceptions; - } - - @Override - public void run() { - try { - final MessageDigest messageDigest = MessageDigest.getInstance(algorithm); - checkManifestEntry(entry, messageDigest, algorithm); - } catch (IOException | CorruptChecksumException | NoSuchAlgorithmException e) { - exceptions.add(e); - } - latch.countDown(); - } - - protected static void checkManifestEntry(final Entry entry, final MessageDigest messageDigest, final String algorithm) throws IOException, CorruptChecksumException{ - if(Files.exists(entry.getKey())){ - logger.debug(messages.getString("checking_checksums"), entry.getKey(), entry.getValue()); - final String hash = Hasher.hash(entry.getKey(), messageDigest); - logger.debug("computed hash [{}] for file [{}]", hash, entry.getKey()); - if(!hash.equals(entry.getValue())){ - throw new CorruptChecksumException(messages.getString("corrupt_checksum_error"), entry.getKey(), algorithm, entry.getValue(), hash); - } - } - //if the file doesn't exist it will be caught by checkAllFilesListedInManifestExist method - } -} diff --git a/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java b/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java deleted file mode 100644 index 0bf8cd0e3..000000000 --- a/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java +++ /dev/null @@ -1,34 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Set; - -import org.slf4j.helpers.MessageFormatter; - -import gov.loc.repository.bagit.exceptions.FileNotInManifestException; - -/** - * Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests. - */ -public class PayloadFileExistsInAtLeastOneManifestVistor extends AbstractPayloadFileExistsInManifestsVistor { - private transient final Set filesListedInManifests; - - public PayloadFileExistsInAtLeastOneManifestVistor(final Set filesListedInManifests, final boolean ignoreHiddenFiles) { - super(ignoreHiddenFiles); - this.filesListedInManifests = filesListedInManifests; - } - - @Override - public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs)throws FileNotInManifestException{ - if(Files.isRegularFile(path) && !filesListedInManifests.contains(path.normalize())){ - final String formattedMessage = messages.getString("file_not_in_any_manifest_error"); - throw new FileNotInManifestException(MessageFormatter.format(formattedMessage, path).getMessage()); - } - logger.debug("[{}] is in at least one manifest", path); - return FileVisitResult.CONTINUE; - } -} diff --git a/src/main/java/gov/loc/repository/bagit/verify/PayloadVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/PayloadVerifier.java deleted file mode 100644 index 2a7fd884c..000000000 --- a/src/main/java/gov/loc/repository/bagit/verify/PayloadVerifier.java +++ /dev/null @@ -1,179 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.ResourceBundle; -import java.util.Set; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.helpers.MessageFormatter; - -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.reader.ManifestReader; -import gov.loc.repository.bagit.util.PathUtils; - -/** - * Responsible for all things related to the manifest during verification. - */ -public class PayloadVerifier implements AutoCloseable{ - private static final Logger logger = LoggerFactory.getLogger(PayloadVerifier.class); - private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); - - private transient final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping; - private transient final ExecutorService executor; - - /** - * Create a PayloadVerifier using a cached thread pool and the - * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} mapping - */ - public PayloadVerifier(){ - this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), Executors.newCachedThreadPool()); - } - - /** - * Create a PayloadVerifier using a cached thread pool and a custom mapping - * - * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm - */ - public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping) { - this(nameMapping, Executors.newCachedThreadPool()); - } - - /** - * Create a PayloadVerifier using a custom thread pool and the - * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} mapping - * - * @param executor the thread pool to use when doing work - */ - public PayloadVerifier(final ExecutorService executor) { - this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), executor); - } - - /** - * Create a PayloadVerifier using a custom thread pool and a custom mapping - * - * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm - * @param executor the thread pool to use when doing work - */ - public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping, final ExecutorService executor) { - this.nameMapping = nameMapping; - this.executor = executor; - } - - @Override - public void close() throws SecurityException{ - //shutdown the thread pool so the resource isn't leaked - executor.shutdown(); - } - - /** - * Verify that all the files in the payload directory are listed in the manifest and - * all files listed in the manifests exist. - * - * @param bag the bag to check to check - * @param ignoreHiddenFiles to ignore hidden files unless they are specifically listed in a manifest - * @throws IOException if there is a problem reading a file - * @throws MaliciousPathException the path in the manifest was specifically crafted to cause harm - * @throws UnsupportedAlgorithmException if the algorithm used for the manifest is unsupported - * @throws InvalidBagitFileFormatException if any of the manifests don't conform to the bagit specification - * @throws FileNotInPayloadDirectoryException if a file is listed in a manifest but doesn't exist in the payload directory - * @throws InterruptedException if a thread is interrupted while doing work - */ - public void verifyPayload(final Bag bag, final boolean ignoreHiddenFiles) - throws IOException, MaliciousPathException, UnsupportedAlgorithmException, - InvalidBagitFileFormatException, FileNotInPayloadDirectoryException, InterruptedException { - - final Set allFilesListedInManifests = getAllFilesListedInManifests(bag); - checkAllFilesListedInManifestExist(allFilesListedInManifests); - - if (bag.getVersion().isOlder(new Version(1, 0))) { - checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(allFilesListedInManifests, PathUtils.getDataDir(bag), ignoreHiddenFiles); - } else { - CheckAllFilesInPayloadDirAreListedInAllManifests(bag.getPayLoadManifests(), PathUtils.getDataDir(bag), ignoreHiddenFiles); - } - } - - /* - * get all the files listed in all the manifests - */ - private Set getAllFilesListedInManifests(final Bag bag) - throws IOException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException { - logger.debug(messages.getString("all_files_in_manifests")); - final Set filesListedInManifests = new HashSet<>(); - - try(DirectoryStream directoryStream = - Files.newDirectoryStream(PathUtils.getBagitDir(bag.getVersion(), bag.getRootDir()))){ - for (final Path path : directoryStream) { - final String filename = PathUtils.getFilename(path); - if (filename.startsWith("tagmanifest-") || filename.startsWith("manifest-")) { - logger.debug(messages.getString("get_listing_in_manifest"), path); - final Manifest manifest = ManifestReader.readManifest(nameMapping, path, bag.getRootDir(), - bag.getFileEncoding()); - filesListedInManifests.addAll(manifest.getFileToChecksumMap().keySet()); - } - } - } - - return filesListedInManifests; - } - - /* - * Make sure all the listed files actually exist - */ - @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") - private void checkAllFilesListedInManifestExist(final Set files) throws FileNotInPayloadDirectoryException, InterruptedException { - final CountDownLatch latch = new CountDownLatch(files.size()); - final Set missingFiles = new ConcurrentSkipListSet<>(); - - logger.info(messages.getString("check_all_files_in_manifests_exist")); - for (final Path file : files) { - executor.execute(new CheckIfFileExistsTask(file, missingFiles, latch)); - } - - latch.await(); - - if (!missingFiles.isEmpty()) { - final String formattedMessage = messages.getString("missing_payload_files_error"); - throw new FileNotInPayloadDirectoryException(MessageFormatter.format(formattedMessage, missingFiles).getMessage()); - } - } - - /* - * Make sure all files in the directory are in at least 1 manifest - */ - private static void checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(final Set filesListedInManifests, - final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException { - logger.debug(messages.getString("checking_file_in_at_least_one_manifest"), payloadDir); - if (Files.exists(payloadDir)) { - Files.walkFileTree(payloadDir, - new PayloadFileExistsInAtLeastOneManifestVistor(filesListedInManifests, ignoreHiddenFiles)); - } - } - - /* - * as per the bagit-spec 1.0+ all files have to be listed in all manifests - */ - private static void CheckAllFilesInPayloadDirAreListedInAllManifests(final Set payLoadManifests, - final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException { - logger.debug(messages.getString("checking_file_in_all_manifests"), payloadDir); - if (Files.exists(payloadDir)) { - Files.walkFileTree(payloadDir, new PayloadFileExistsInAllManifestsVistor(payLoadManifests, ignoreHiddenFiles)); - } - } -} diff --git a/src/main/java/gov/loc/repository/bagit/writer/BagitFileWriter.java b/src/main/java/gov/loc/repository/bagit/writer/BagitFileWriter.java deleted file mode 100644 index 41aedb7b9..000000000 --- a/src/main/java/gov/loc/repository/bagit/writer/BagitFileWriter.java +++ /dev/null @@ -1,49 +0,0 @@ -package gov.loc.repository.bagit.writer; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ResourceBundle; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import gov.loc.repository.bagit.domain.Version; - -/** - * Responsible for writing the bagit.txt to the filesystem - */ -public final class BagitFileWriter { - private static final Logger logger = LoggerFactory.getLogger(BagitFileWriter.class); - private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); - - private BagitFileWriter(){ - //intentionally left empty - } - - /** - * Write the bagit.txt file in required UTF-8 encoding. - * - * @param version the version of the bag to write out - * @param encoding the encoding of the tag files - * @param outputDir the root of the bag - * - * @throws IOException if there was a problem writing the file - */ - public static void writeBagitFile(final Version version, final Charset encoding, final Path outputDir) throws IOException{ - final Path bagitPath = outputDir.resolve("bagit.txt"); - logger.debug(messages.getString("write_bagit_file_to_path"), outputDir); - - final String firstLine = "BagIt-Version: " + version + System.lineSeparator(); - logger.debug(messages.getString("writing_line_to_file"), firstLine, bagitPath); - Files.write(bagitPath, firstLine.getBytes(StandardCharsets.UTF_8), - StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); - - final String secondLine = "Tag-File-Character-Encoding: " + encoding + System.lineSeparator(); - logger.debug(messages.getString("writing_line_to_file"), secondLine, bagitPath); - Files.write(bagitPath, secondLine.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.APPEND); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/writer/MetadataWriter.java b/src/main/java/gov/loc/repository/bagit/writer/MetadataWriter.java deleted file mode 100644 index 5f971d41f..000000000 --- a/src/main/java/gov/loc/repository/bagit/writer/MetadataWriter.java +++ /dev/null @@ -1,58 +0,0 @@ -package gov.loc.repository.bagit.writer; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ResourceBundle; -import java.util.AbstractMap.SimpleImmutableEntry; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import gov.loc.repository.bagit.domain.Metadata; -import gov.loc.repository.bagit.domain.Version; - -/** - * Responsible for writing out the bag {@link Metadata} to the filesystem - */ -public final class MetadataWriter { - private static final Logger logger = LoggerFactory.getLogger(MetadataWriter.class); - private static final Version VERSION_0_95 = new Version(0, 95); - private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); - - private MetadataWriter(){ - //intentionall left empty - } - - /** - * Write the bag-info.txt (or package-info.txt) file to the specified outputDir with specified encoding (charsetName) - * - * @param metadata the key value pair info in the bag-info.txt file - * @param version the version of the bag you are writing - * @param outputDir the root of the bag - * @param charsetName the name of the encoding for the file - * - * @throws IOException if there was a problem writing a file - */ - public static void writeBagMetadata(final Metadata metadata, final Version version, final Path outputDir, final Charset charsetName) throws IOException{ - Path bagInfoFilePath = outputDir.resolve("bag-info.txt"); - if(version.isSameOrOlder(VERSION_0_95)){ - bagInfoFilePath = outputDir.resolve("package-info.txt"); - } - logger.debug(messages.getString("writing_metadata_to_path"), bagInfoFilePath.getFileName(), outputDir); - - Files.deleteIfExists(bagInfoFilePath); - final StringBuilder lines = new StringBuilder(); - - for(final SimpleImmutableEntry entry : metadata.getAll()){ - final String line = entry.getKey() + ": " + entry.getValue() + System.lineSeparator(); - lines.append(line); - } - - logger.debug(messages.getString("writing_line_to_file"), lines.toString(), bagInfoFilePath); - Files.write(bagInfoFilePath, lines.toString().getBytes(charsetName), - StandardOpenOption.APPEND, StandardOpenOption.CREATE); - } -} diff --git a/src/main/java/gov/loc/repository/bagit/annotation/Incubating.java b/src/main/java/nl/knaw/dans/bagit/annotation/Incubating.java similarity index 54% rename from src/main/java/gov/loc/repository/bagit/annotation/Incubating.java rename to src/main/java/nl/knaw/dans/bagit/annotation/Incubating.java index 8e9e49ab5..b344ca2e6 100644 --- a/src/main/java/gov/loc/repository/bagit/annotation/Incubating.java +++ b/src/main/java/nl/knaw/dans/bagit/annotation/Incubating.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.annotation; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.annotation; import java.lang.annotation.Documented; import java.lang.annotation.Retention; diff --git a/src/main/java/gov/loc/repository/bagit/conformance/BagLinter.java b/src/main/java/nl/knaw/dans/bagit/conformance/BagLinter.java similarity index 80% rename from src/main/java/gov/loc/repository/bagit/conformance/BagLinter.java rename to src/main/java/nl/knaw/dans/bagit/conformance/BagLinter.java index 6221048c4..44b71ee18 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/BagLinter.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/BagLinter.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.io.IOException; import java.io.InputStream; @@ -14,30 +29,30 @@ import java.util.ResourceBundle; import java.util.Set; +import nl.knaw.dans.bagit.conformance.profile.BagitProfile; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.exceptions.InvalidBagMetadataException; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.exceptions.UnparsableVersionException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; +import nl.knaw.dans.bagit.reader.BagitTextFileReader; +import nl.knaw.dans.bagit.reader.KeyValueReader; +import nl.knaw.dans.bagit.verify.BagVerifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; -import gov.loc.repository.bagit.conformance.profile.BagitProfile; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.InvalidBagMetadataException; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.UnparsableVersionException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.exceptions.conformance.BagitVersionIsNotAcceptableException; -import gov.loc.repository.bagit.exceptions.conformance.FetchFileNotAllowedException; -import gov.loc.repository.bagit.exceptions.conformance.MetatdataValueIsNotAcceptableException; -import gov.loc.repository.bagit.exceptions.conformance.MetatdataValueIsNotRepeatableException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredManifestNotPresentException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredMetadataFieldNotPresentException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredTagFileNotPresentException; -import gov.loc.repository.bagit.reader.BagitTextFileReader; -import gov.loc.repository.bagit.reader.KeyValueReader; -import gov.loc.repository.bagit.verify.BagVerifier; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.exceptions.conformance.BagitVersionIsNotAcceptableException; +import nl.knaw.dans.bagit.exceptions.conformance.FetchFileNotAllowedException; +import nl.knaw.dans.bagit.exceptions.conformance.MetatdataValueIsNotAcceptableException; +import nl.knaw.dans.bagit.exceptions.conformance.MetatdataValueIsNotRepeatableException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredManifestNotPresentException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredMetadataFieldNotPresentException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredTagFileNotPresentException; /** * Responsible for checking a bag and providing insight into how it cause problems. @@ -73,7 +88,7 @@ private BagLinter(){ * @throws BagitVersionIsNotAcceptableException if the version of the bag is not in the list of acceptable versions * @throws RequiredTagFileNotPresentException if a tag file is not present but should be */ - public static void checkAgainstProfile(final InputStream jsonProfile, final Bag bag) throws JsonParseException, JsonMappingException, + public static void checkAgainstProfile(final InputStream jsonProfile, final Bag bag) throws JsonParseException, JsonMappingException, IOException, FetchFileNotAllowedException, RequiredMetadataFieldNotPresentException, MetatdataValueIsNotAcceptableException, RequiredManifestNotPresentException, BagitVersionIsNotAcceptableException, RequiredTagFileNotPresentException, MetatdataValueIsNotRepeatableException{ BagProfileChecker.bagConformsToProfile(jsonProfile, bag); diff --git a/src/main/java/gov/loc/repository/bagit/conformance/BagProfileChecker.java b/src/main/java/nl/knaw/dans/bagit/conformance/BagProfileChecker.java similarity index 83% rename from src/main/java/gov/loc/repository/bagit/conformance/BagProfileChecker.java rename to src/main/java/nl/knaw/dans/bagit/conformance/BagProfileChecker.java index bec5709db..e2d9b2471 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/BagProfileChecker.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/BagProfileChecker.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.io.IOException; import java.io.InputStream; @@ -11,6 +26,12 @@ import java.util.ResourceBundle; import java.util.Set; +import nl.knaw.dans.bagit.conformance.profile.BagInfoRequirement; +import nl.knaw.dans.bagit.conformance.profile.BagitProfile; +import nl.knaw.dans.bagit.conformance.profile.BagitProfileDeserializer; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.domain.Manifest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; @@ -20,20 +41,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import gov.loc.repository.bagit.conformance.profile.BagInfoRequirement; -import gov.loc.repository.bagit.conformance.profile.BagitProfile; -import gov.loc.repository.bagit.conformance.profile.BagitProfileDeserializer; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.FetchItem; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.domain.Metadata; -import gov.loc.repository.bagit.exceptions.conformance.BagitVersionIsNotAcceptableException; -import gov.loc.repository.bagit.exceptions.conformance.FetchFileNotAllowedException; -import gov.loc.repository.bagit.exceptions.conformance.MetatdataValueIsNotAcceptableException; -import gov.loc.repository.bagit.exceptions.conformance.MetatdataValueIsNotRepeatableException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredManifestNotPresentException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredMetadataFieldNotPresentException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredTagFileNotPresentException; +import nl.knaw.dans.bagit.domain.Metadata; +import nl.knaw.dans.bagit.exceptions.conformance.BagitVersionIsNotAcceptableException; +import nl.knaw.dans.bagit.exceptions.conformance.FetchFileNotAllowedException; +import nl.knaw.dans.bagit.exceptions.conformance.MetatdataValueIsNotAcceptableException; +import nl.knaw.dans.bagit.exceptions.conformance.MetatdataValueIsNotRepeatableException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredManifestNotPresentException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredMetadataFieldNotPresentException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredTagFileNotPresentException; /** * Responsible for checking a bag against a profile @@ -66,7 +81,7 @@ private BagProfileChecker(){ * @throws BagitVersionIsNotAcceptableException if the version of the bag is not in the list of acceptable versions * @throws RequiredTagFileNotPresentException if a tag file is not present but should be */ - public static void bagConformsToProfile(final InputStream jsonProfile, final Bag bag) throws JsonParseException, JsonMappingException, + public static void bagConformsToProfile(final InputStream jsonProfile, final Bag bag) throws JsonParseException, JsonMappingException, IOException, FetchFileNotAllowedException, RequiredMetadataFieldNotPresentException, MetatdataValueIsNotAcceptableException, RequiredManifestNotPresentException, BagitVersionIsNotAcceptableException, RequiredTagFileNotPresentException, MetatdataValueIsNotRepeatableException{ @@ -102,7 +117,7 @@ private static void checkFetch(final Path rootDir, final boolean allowFetchFile, } } - private static void checkMetadata(final Metadata bagMetadata, final Map bagInfoEntryRequirements) + private static void checkMetadata(final Metadata bagMetadata, final Map bagInfoEntryRequirements) throws RequiredMetadataFieldNotPresentException, MetatdataValueIsNotAcceptableException, MetatdataValueIsNotRepeatableException{ for(final Entry bagInfoEntryRequirement : bagInfoEntryRequirements.entrySet()){ @@ -157,11 +172,11 @@ private static void requiredManifestsExist(final Set manifests, final for(final String requiredManifestType : requiredManifestTypes){ if(!manifestTypesPresent.contains(requiredManifestType)){ final StringBuilder sb = new StringBuilder(); - if(isPayloadManifest){ sb.append("tag"); - sb.append(MessageFormatter.format(messages.getString("required_tag_manifest_type_not_present"), requiredManifestType).getMessage()); + if(isPayloadManifest){ + sb.append(MessageFormatter.format(messages.getString("required_manifest_type_not_present"), requiredManifestType).getMessage()); } else{ - sb.append(MessageFormatter.format(messages.getString("required_manifest_type_not_present"), requiredManifestType).getMessage()); + sb.append(MessageFormatter.format(messages.getString("required_tag_manifest_type_not_present"), requiredManifestType).getMessage()); } throw new RequiredManifestNotPresentException(sb.toString()); diff --git a/src/main/java/gov/loc/repository/bagit/conformance/BagitWarning.java b/src/main/java/nl/knaw/dans/bagit/conformance/BagitWarning.java similarity index 66% rename from src/main/java/gov/loc/repository/bagit/conformance/BagitWarning.java rename to src/main/java/nl/knaw/dans/bagit/conformance/BagitWarning.java index f68332a1a..3fef9407a 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/BagitWarning.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/BagitWarning.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.util.ResourceBundle; diff --git a/src/main/java/gov/loc/repository/bagit/conformance/EncodingChecker.java b/src/main/java/nl/knaw/dans/bagit/conformance/EncodingChecker.java similarity index 59% rename from src/main/java/gov/loc/repository/bagit/conformance/EncodingChecker.java rename to src/main/java/nl/knaw/dans/bagit/conformance/EncodingChecker.java index b878ad6d0..7a81cd3be 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/EncodingChecker.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/EncodingChecker.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; diff --git a/src/main/java/gov/loc/repository/bagit/conformance/ManifestChecker.java b/src/main/java/nl/knaw/dans/bagit/conformance/ManifestChecker.java similarity index 91% rename from src/main/java/gov/loc/repository/bagit/conformance/ManifestChecker.java rename to src/main/java/nl/knaw/dans/bagit/conformance/ManifestChecker.java index af9438c26..a70e7294d 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/ManifestChecker.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/ManifestChecker.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.io.BufferedReader; import java.io.IOException; @@ -14,18 +29,18 @@ import java.util.ResourceBundle; import java.util.Set; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; +import nl.knaw.dans.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.reader.ManifestReader; +import nl.knaw.dans.bagit.util.PathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.reader.ManifestReader; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.domain.Version; /** * Part of the BagIt conformance suite. @@ -123,9 +138,10 @@ private static void checkManifestPayload(final Path manifestFile, final Charset String path = parsePath(line); path = checkForManifestCreatedWithMD5SumTools(path, warnings, warningsToIgnore); - paths.add(path.toLowerCase()); checkForDifferentCase(path, paths, manifestFile, warnings, warningsToIgnore); + paths.add(path.toLowerCase()); + if(encoding.name().startsWith("UTF")){ checkNormalization(path, manifestFile.getParent(), warnings, warningsToIgnore); } @@ -256,7 +272,7 @@ static void checkAlgorthm(final String algorithm, final Set warnin warnings.add(BagitWarning.WEAK_CHECKSUM_ALGORITHM); } - else if(!warningsToIgnore.contains(BagitWarning.NON_STANDARD_ALGORITHM) && !"SHA-512".equals(upperCaseAlg)){ + else if(!warningsToIgnore.contains(BagitWarning.NON_STANDARD_ALGORITHM) && !"SHA512".equals(upperCaseAlg)){ logger.warn(messages.getString("non_standard_algorithm_warning"), algorithm); warnings.add(BagitWarning.NON_STANDARD_ALGORITHM); } diff --git a/src/main/java/gov/loc/repository/bagit/conformance/MetadataChecker.java b/src/main/java/nl/knaw/dans/bagit/conformance/MetadataChecker.java similarity index 69% rename from src/main/java/gov/loc/repository/bagit/conformance/MetadataChecker.java rename to src/main/java/nl/knaw/dans/bagit/conformance/MetadataChecker.java index 912287fb1..0d02214e5 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/MetadataChecker.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/MetadataChecker.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.io.IOException; import java.nio.charset.Charset; @@ -9,12 +24,11 @@ import java.util.ResourceBundle; import java.util.Set; +import nl.knaw.dans.bagit.exceptions.InvalidBagMetadataException; +import nl.knaw.dans.bagit.reader.MetadataReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.exceptions.InvalidBagMetadataException; -import gov.loc.repository.bagit.reader.MetadataReader; - /** * Part of the BagIt conformance suite. * This checker checks the bag metadata (bag-info.txt) for various problems. @@ -28,7 +42,7 @@ private MetadataChecker(){ } public static void checkBagMetadata(final Path bagitDir, final Charset encoding, final Set warnings, - final Collection warningsToIgnore) throws IOException, InvalidBagMetadataException{ + final Collection warningsToIgnore) throws IOException, InvalidBagMetadataException { checkForPayloadOxumMetadata(bagitDir, encoding, warnings, warningsToIgnore); } diff --git a/src/main/java/gov/loc/repository/bagit/conformance/VersionChecker.java b/src/main/java/nl/knaw/dans/bagit/conformance/VersionChecker.java similarity index 57% rename from src/main/java/gov/loc/repository/bagit/conformance/VersionChecker.java rename to src/main/java/nl/knaw/dans/bagit/conformance/VersionChecker.java index df8d1ab29..a399d0408 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/VersionChecker.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/VersionChecker.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.util.Collection; import java.util.ResourceBundle; @@ -7,7 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.Version; +import nl.knaw.dans.bagit.domain.Version; /** * Part of the BagIt conformance suite. diff --git a/src/main/java/gov/loc/repository/bagit/conformance/profile/BagInfoRequirement.java b/src/main/java/nl/knaw/dans/bagit/conformance/profile/BagInfoRequirement.java similarity index 54% rename from src/main/java/gov/loc/repository/bagit/conformance/profile/BagInfoRequirement.java rename to src/main/java/nl/knaw/dans/bagit/conformance/profile/BagInfoRequirement.java index 155c2ab03..145b6f329 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/profile/BagInfoRequirement.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/profile/BagInfoRequirement.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance.profile; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; import java.util.ArrayList; import java.util.List; @@ -10,7 +25,7 @@ public class BagInfoRequirement { private boolean required; private List acceptableValues = new ArrayList<>(); - private boolean repeatable; + private boolean repeatable = true; @Override public boolean equals(final Object other) { @@ -31,12 +46,28 @@ public int hashCode() { public BagInfoRequirement(){ //intentionally left empty } - + /** + * Constructs a new BagInfoRequirement setting {@link #repeatable} to true (default). + * @param required Indicates whether or not the tag is required. + * @param acceptableValues List of acceptable values. + */ public BagInfoRequirement(final boolean required, final List acceptableValues){ this.required = required; this.acceptableValues = acceptableValues; } + /** + * Constructs a new BagInfoRequirement. + * @param required Indicates whether or not the tag is required. + * @param acceptableValues List of acceptable values. + * @param repeatable Indicates whether or not the tag is repeatable. + */ + public BagInfoRequirement(final boolean required, final List acceptableValues, final boolean repeatable){ + this.required = required; + this.acceptableValues = acceptableValues; + this.repeatable = repeatable; + } + @Override public String toString() { return "[required=" + required + ", acceptableValues=" + acceptableValues + ", repeatable=" + repeatable + "]"; diff --git a/src/main/java/gov/loc/repository/bagit/conformance/profile/BagitProfile.java b/src/main/java/nl/knaw/dans/bagit/conformance/profile/BagitProfile.java similarity index 82% rename from src/main/java/gov/loc/repository/bagit/conformance/profile/BagitProfile.java rename to src/main/java/nl/knaw/dans/bagit/conformance/profile/BagitProfile.java index 1ed5158a9..2eef5ef08 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/profile/BagitProfile.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/profile/BagitProfile.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance.profile; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; import java.util.ArrayList; import java.util.HashMap; @@ -16,6 +31,7 @@ public class BagitProfile { private String externalDescription = ""; private String contactName = ""; private String contactEmail = ""; + private String contactPhone = ""; private String version = ""; private Map bagInfoRequirements = new HashMap<>(); @@ -38,6 +54,7 @@ public boolean equals(final Object other) { && Objects.equals(sourceOrganization, castOther.sourceOrganization) && Objects.equals(externalDescription, castOther.externalDescription) && Objects.equals(contactName, castOther.contactName) && Objects.equals(contactEmail, castOther.contactEmail) + && Objects.equals(contactPhone, castOther.contactPhone) && Objects.equals(version, castOther.version) && Objects.equals(bagInfoRequirements, castOther.bagInfoRequirements) && Objects.equals(manifestTypesRequired, castOther.manifestTypesRequired) @@ -50,15 +67,14 @@ public boolean equals(final Object other) { } @Override public int hashCode() { - return Objects.hash(bagitProfileIdentifier, sourceOrganization, externalDescription, contactName, contactEmail, - version, bagInfoRequirements, manifestTypesRequired, fetchFileAllowed, serialization, + return Objects.hash(bagitProfileIdentifier, sourceOrganization, externalDescription, contactName, contactEmail, contactPhone, version, bagInfoRequirements, manifestTypesRequired, fetchFileAllowed, serialization, acceptableMIMESerializationTypes, acceptableBagitVersions, tagManifestTypesRequired, tagFilesRequired); } @Override public String toString() { return "BagitProfile [bagitProfileIdentifier=" + bagitProfileIdentifier + ", sourceOrganization=" + sourceOrganization + ", externalDescription=" + externalDescription + ", contactName=" + contactName - + ", contactEmail=" + contactEmail + ", version=" + version + ", bagInfoRequirements=" + bagInfoRequirements + + ", contactEmail=" + contactEmail + ", contactPhone=" + contactPhone + ", version=" + version + ", bagInfoRequirements=" + bagInfoRequirements + ", manifestTypesRequired=" + manifestTypesRequired + ", fetchFileAllowed=" + fetchFileAllowed + ", serialization=" + serialization + ", acceptableMIMESerializationTypes=" + acceptableMIMESerializationTypes + ", acceptableBagitVersions=" + acceptableBagitVersions + ", tagManifestTypesRequired=" @@ -143,6 +159,12 @@ public String getContactEmail() { public void setContactEmail(final String contactEmail) { this.contactEmail = contactEmail; } + public String getContactPhone() { + return contactPhone; + } + public void setContactPhone(final String contactPhone) { + this.contactPhone = contactPhone; + } public String getVersion() { return version; } diff --git a/src/main/java/gov/loc/repository/bagit/conformance/profile/BagitProfileDeserializer.java b/src/main/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileDeserializer.java similarity index 57% rename from src/main/java/gov/loc/repository/bagit/conformance/profile/BagitProfileDeserializer.java rename to src/main/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileDeserializer.java index 4684d02bc..aced4aa15 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/profile/BagitProfileDeserializer.java +++ b/src/main/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileDeserializer.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance.profile; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; import java.io.IOException; import java.util.ArrayList; @@ -19,9 +34,10 @@ import com.fasterxml.jackson.databind.deser.std.StdDeserializer; /** - * Deserialize bagit profile json to a {@link BagitProfile} + * Deserialize bagit profile json to a {@link BagitProfile} */ public class BagitProfileDeserializer extends StdDeserializer { + private static final long serialVersionUID = 1L; private static final Logger logger = LoggerFactory.getLogger(BagitProfileDeserializer.class); private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); @@ -36,149 +52,198 @@ public BagitProfileDeserializer(final Class vc) { @Override public BagitProfile deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException, JsonProcessingException { + throws IOException, JsonProcessingException { final BagitProfile profile = new BagitProfile(); final JsonNode node = p.getCodec().readTree(p); - + parseBagitProfileInfo(node, profile); - + profile.setBagInfoRequirements(parseBagInfo(node)); - + profile.getManifestTypesRequired().addAll(parseManifestTypesRequired(node)); - + profile.setFetchFileAllowed(node.get("Allow-Fetch.txt").asBoolean()); logger.debug(messages.getString("fetch_allowed"), profile.isFetchFileAllowed()); - + profile.setSerialization(Serialization.valueOf(node.get("Serialization").asText())); - logger.debug(messages.getString("serialization_allowed"),profile.getSerialization()); - + logger.debug(messages.getString("serialization_allowed"), profile.getSerialization()); + profile.getAcceptableMIMESerializationTypes().addAll(parseAcceptableSerializationFormats(node)); - + profile.getTagManifestTypesRequired().addAll(parseRequiredTagmanifestTypes(node)); - + profile.getTagFilesRequired().addAll(parseRequiredTagFiles(node)); - + profile.getAcceptableBagitVersions().addAll(parseAcceptableVersions(node)); - + return profile; } - - private static void parseBagitProfileInfo(final JsonNode node, final BagitProfile profile){ - final JsonNode bagitProfileInfoNode = node.get("BagIt-Profile-Info"); + + private static void parseBagitProfileInfo(final JsonNode node, final BagitProfile profile) { logger.debug(messages.getString("parsing_bagit_profile_info_section")); + final JsonNode bagitProfileInfoNode = node.get("BagIt-Profile-Info"); + parseMandatoryTagsOfBagitProfileInfo(bagitProfileInfoNode, profile); + parseOptionalTagsOfBagitProfileInfo(bagitProfileInfoNode, profile); + } + + /** + * Parse required tags due to specification defined at bagit profiles + * Note: If one of the tags is missing, a NullPointerException is thrown. + * + * @param bagitProfileInfoNode Root node of the bagit profile info section. + * @param profile Representation of bagit profile. + */ + private static void parseMandatoryTagsOfBagitProfileInfo(final JsonNode bagitProfileInfoNode, final BagitProfile profile) { + logger.debug(messages.getString("parsing_mandatory_tags_of_bagit_profile_info_section")); + final String profileIdentifier = bagitProfileInfoNode.get("BagIt-Profile-Identifier").asText(); logger.debug(messages.getString("identifier"), profileIdentifier); profile.setBagitProfileIdentifier(profileIdentifier); - + final String sourceOrg = bagitProfileInfoNode.get("Source-Organization").asText(); logger.debug(messages.getString("source_organization"), sourceOrg); profile.setSourceOrganization(sourceOrg); - - final String contactName = bagitProfileInfoNode.get("Contact-Name").asText(); - logger.debug(messages.getString("contact_name"), contactName); - profile.setContactName(contactName); - - final String contactEmail = bagitProfileInfoNode.get("Contact-Email").asText(); - logger.debug(messages.getString("contact_email"), contactEmail); - profile.setContactEmail(contactEmail); - + final String extDescript = bagitProfileInfoNode.get("External-Description").asText(); logger.debug(messages.getString("external_description"), extDescript); profile.setExternalDescription(extDescript); - + final String version = bagitProfileInfoNode.get("Version").asText(); logger.debug(messages.getString("version"), version); profile.setVersion(version); } - + + /** + * Parse optional tags due to specification defined at bagit profiles + * + * @param bagitProfileInfoNode Root node of the bagit profile info section. + * @param profile Representation of bagit profile . + */ + private static void parseOptionalTagsOfBagitProfileInfo(final JsonNode bagitProfileInfoNode, final BagitProfile profile) { + logger.debug(messages.getString("parsing_optional_tags_of_bagit_profile_info_section")); + + final JsonNode contactNameNode = bagitProfileInfoNode.get("Contact-Name"); + if (contactNameNode != null) { + final String contactName = contactNameNode.asText(); + logger.debug(messages.getString("contact_name"), contactName); + profile.setContactName(contactName); + } + + final JsonNode contactEmailNode = bagitProfileInfoNode.get("Contact-Email"); + if (contactEmailNode != null) { + final String contactEmail = contactEmailNode.asText(); + logger.debug(messages.getString("contact_email"), contactEmail); + profile.setContactEmail(contactEmail); + } + + final JsonNode contactPhoneNode = bagitProfileInfoNode.get("Contact-Phone"); + if (contactPhoneNode != null) { + final String contactPhone = contactPhoneNode.asText(); + logger.debug(messages.getString("contact_phone"), contactPhone); + profile.setContactPhone(contactPhone); + } + } + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") - private static Map parseBagInfo(final JsonNode rootNode){ + private static Map parseBagInfo(final JsonNode rootNode) { final JsonNode bagInfoNode = rootNode.get("Bag-Info"); logger.debug(messages.getString("parsing_bag_info")); - final Map bagInfo = new HashMap<>(); - + final Map bagInfo = new HashMap<>(); + final Iterator> nodes = bagInfoNode.fields(); //stuck in java 6... - - while(nodes.hasNext()){ + + while (nodes.hasNext()) { final Entry node = nodes.next(); - + final BagInfoRequirement entry = new BagInfoRequirement(); - entry.setRequired(node.getValue().get("required").asBoolean()); - + // due to specification required is false by default. + final JsonNode requiredNode = node.getValue().get("required"); + if (requiredNode != null) { + entry.setRequired(requiredNode.asBoolean()); + } + final JsonNode valuesNode = node.getValue().get("values"); - if(valuesNode != null){ - for(final JsonNode value : valuesNode){ + if (valuesNode != null) { + for (final JsonNode value : valuesNode) { entry.getAcceptableValues().add(value.asText()); } } - + + final JsonNode repeatableNode = node.getValue().get("repeatable"); + if (repeatableNode != null) { + entry.setRepeatable(repeatableNode.asBoolean()); + } + logger.debug("{}: {}", node.getKey(), entry); bagInfo.put(node.getKey(), entry); } - + return bagInfo; } - - private static List parseManifestTypesRequired(final JsonNode node){ + + private static List parseManifestTypesRequired(final JsonNode node) { final JsonNode manifests = node.get("Manifests-Required"); - + final List manifestTypes = new ArrayList<>(); - + for (final JsonNode manifestName : manifests) { manifestTypes.add(manifestName.asText()); } - + logger.debug(messages.getString("required_manifest_types"), manifestTypes); - + return manifestTypes; } - - private static List parseAcceptableSerializationFormats(final JsonNode node){ + + private static List parseAcceptableSerializationFormats(final JsonNode node) { final JsonNode serialiationFormats = node.get("Accept-Serialization"); final List serialTypes = new ArrayList<>(); - + for (final JsonNode serialiationFormat : serialiationFormats) { serialTypes.add(serialiationFormat.asText()); } logger.debug(messages.getString("acceptable_serialization_mime_types"), serialTypes); - + return serialTypes; } - - private static List parseRequiredTagmanifestTypes(final JsonNode node){ + + private static List parseRequiredTagmanifestTypes(final JsonNode node) { final JsonNode tagManifestsRequiredNodes = node.get("Tag-Manifests-Required"); final List requiredTagmanifestTypes = new ArrayList<>(); - - for(final JsonNode tagManifestsRequiredNode : tagManifestsRequiredNodes){ - requiredTagmanifestTypes.add(tagManifestsRequiredNode.asText()); + if (tagManifestsRequiredNodes != null) { + for (final JsonNode tagManifestsRequiredNode : tagManifestsRequiredNodes) { + requiredTagmanifestTypes.add(tagManifestsRequiredNode.asText()); + } } logger.debug(messages.getString("required_tagmanifest_types"), requiredTagmanifestTypes); - + return requiredTagmanifestTypes; } - - private static List parseRequiredTagFiles(final JsonNode node){ + + private static List parseRequiredTagFiles(final JsonNode node) { final JsonNode tagFilesRequiredNodes = node.get("Tag-Files-Required"); final List requiredTagFiles = new ArrayList<>(); - - for(final JsonNode tagFilesRequiredNode : tagFilesRequiredNodes){ - requiredTagFiles.add(tagFilesRequiredNode.asText()); + + if (tagFilesRequiredNodes != null) { + for (final JsonNode tagFilesRequiredNode : tagFilesRequiredNodes) { + requiredTagFiles.add(tagFilesRequiredNode.asText()); + } } logger.debug(messages.getString("tag_files_required"), requiredTagFiles); - + return requiredTagFiles; } - - private static List parseAcceptableVersions(final JsonNode node){ + + private static List parseAcceptableVersions(final JsonNode node) { final JsonNode acceptableVersionsNodes = node.get("Accept-BagIt-Version"); final List acceptableVersions = new ArrayList<>(); - - for(final JsonNode acceptableVersionsNode : acceptableVersionsNodes){ + + for (final JsonNode acceptableVersionsNode : acceptableVersionsNodes) { acceptableVersions.add(acceptableVersionsNode.asText()); } logger.debug(messages.getString("acceptable_bagit_versions"), acceptableVersions); - + return acceptableVersions; } } diff --git a/src/main/java/nl/knaw/dans/bagit/conformance/profile/Serialization.java b/src/main/java/nl/knaw/dans/bagit/conformance/profile/Serialization.java new file mode 100644 index 000000000..55be0a147 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/conformance/profile/Serialization.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; + +/** + * The type of serialization required by a {@link BagitProfile} + */ +public enum Serialization { + forbidden, + required, + optional; +} diff --git a/src/main/java/gov/loc/repository/bagit/creator/AbstractCreateManifestsVistor.java b/src/main/java/nl/knaw/dans/bagit/creator/AbstractCreateManifestsVistor.java similarity index 72% rename from src/main/java/gov/loc/repository/bagit/creator/AbstractCreateManifestsVistor.java rename to src/main/java/nl/knaw/dans/bagit/creator/AbstractCreateManifestsVistor.java index d37fd2c25..086c0914d 100644 --- a/src/main/java/gov/loc/repository/bagit/creator/AbstractCreateManifestsVistor.java +++ b/src/main/java/nl/knaw/dans/bagit/creator/AbstractCreateManifestsVistor.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.creator; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.creator; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -10,13 +25,12 @@ import java.util.Map; import java.util.ResourceBundle; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.Hasher; +import nl.knaw.dans.bagit.util.PathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.hash.Hasher; -import gov.loc.repository.bagit.util.PathUtils; - /** * An implementation of the {@link SimpleFileVisitor} class that optionally avoids hidden files. * Mainly used in {@link BagCreator} diff --git a/src/main/java/gov/loc/repository/bagit/creator/BagCreator.java b/src/main/java/nl/knaw/dans/bagit/creator/BagCreator.java similarity index 89% rename from src/main/java/gov/loc/repository/bagit/creator/BagCreator.java rename to src/main/java/nl/knaw/dans/bagit/creator/BagCreator.java index e447fbbc2..2a421cbeb 100644 --- a/src/main/java/gov/loc/repository/bagit/creator/BagCreator.java +++ b/src/main/java/nl/knaw/dans/bagit/creator/BagCreator.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.creator; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.creator; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -13,20 +28,20 @@ import java.util.Map; import java.util.ResourceBundle; +import nl.knaw.dans.bagit.annotation.Incubating; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.Hasher; +import nl.knaw.dans.bagit.hash.SupportedAlgorithm; +import nl.knaw.dans.bagit.util.PathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.annotation.Incubating; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.domain.Metadata; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.hash.Hasher; -import gov.loc.repository.bagit.hash.SupportedAlgorithm; -import gov.loc.repository.bagit.util.PathUtils; -import gov.loc.repository.bagit.writer.BagitFileWriter; -import gov.loc.repository.bagit.writer.ManifestWriter; -import gov.loc.repository.bagit.writer.MetadataWriter; +import nl.knaw.dans.bagit.domain.Metadata; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.writer.BagitFileWriter; +import nl.knaw.dans.bagit.writer.ManifestWriter; +import nl.knaw.dans.bagit.writer.MetadataWriter; /** * Responsible for creating a bag in place. diff --git a/src/main/java/gov/loc/repository/bagit/creator/CreatePayloadManifestsVistor.java b/src/main/java/nl/knaw/dans/bagit/creator/CreatePayloadManifestsVistor.java similarity index 52% rename from src/main/java/gov/loc/repository/bagit/creator/CreatePayloadManifestsVistor.java rename to src/main/java/nl/knaw/dans/bagit/creator/CreatePayloadManifestsVistor.java index 2780418a5..cecd44145 100644 --- a/src/main/java/gov/loc/repository/bagit/creator/CreatePayloadManifestsVistor.java +++ b/src/main/java/nl/knaw/dans/bagit/creator/CreatePayloadManifestsVistor.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.creator; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.creator; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -7,7 +22,7 @@ import java.security.MessageDigest; import java.util.Map; -import gov.loc.repository.bagit.domain.Manifest; +import nl.knaw.dans.bagit.domain.Manifest; /** * Creates the payload manifests by walking the payload files and calculating their checksums diff --git a/src/main/java/gov/loc/repository/bagit/creator/CreateTagManifestsVistor.java b/src/main/java/nl/knaw/dans/bagit/creator/CreateTagManifestsVistor.java similarity index 52% rename from src/main/java/gov/loc/repository/bagit/creator/CreateTagManifestsVistor.java rename to src/main/java/nl/knaw/dans/bagit/creator/CreateTagManifestsVistor.java index 8d96aeb7b..f37a00769 100644 --- a/src/main/java/gov/loc/repository/bagit/creator/CreateTagManifestsVistor.java +++ b/src/main/java/nl/knaw/dans/bagit/creator/CreateTagManifestsVistor.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.creator; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.creator; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -7,7 +22,7 @@ import java.security.MessageDigest; import java.util.Map; -import gov.loc.repository.bagit.domain.Manifest; +import nl.knaw.dans.bagit.domain.Manifest; /** * Creates the tag manifests by walking the tag files and calculating their checksums diff --git a/src/main/java/gov/loc/repository/bagit/domain/Bag.java b/src/main/java/nl/knaw/dans/bagit/domain/Bag.java similarity index 86% rename from src/main/java/gov/loc/repository/bagit/domain/Bag.java rename to src/main/java/nl/knaw/dans/bagit/domain/Bag.java index 9c7185cb3..b3bb2a105 100644 --- a/src/main/java/gov/loc/repository/bagit/domain/Bag.java +++ b/src/main/java/nl/knaw/dans/bagit/domain/Bag.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; diff --git a/src/main/java/gov/loc/repository/bagit/domain/FetchItem.java b/src/main/java/nl/knaw/dans/bagit/domain/FetchItem.java similarity index 75% rename from src/main/java/gov/loc/repository/bagit/domain/FetchItem.java rename to src/main/java/nl/knaw/dans/bagit/domain/FetchItem.java index 9eddb2acc..9b5cfab91 100644 --- a/src/main/java/gov/loc/repository/bagit/domain/FetchItem.java +++ b/src/main/java/nl/knaw/dans/bagit/domain/FetchItem.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.net.URL; import java.nio.file.Path; diff --git a/src/main/java/gov/loc/repository/bagit/domain/Manifest.java b/src/main/java/nl/knaw/dans/bagit/domain/Manifest.java similarity index 65% rename from src/main/java/gov/loc/repository/bagit/domain/Manifest.java rename to src/main/java/nl/knaw/dans/bagit/domain/Manifest.java index 0eeb4e2a2..05e3bd956 100644 --- a/src/main/java/gov/loc/repository/bagit/domain/Manifest.java +++ b/src/main/java/nl/knaw/dans/bagit/domain/Manifest.java @@ -1,12 +1,26 @@ - -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.Objects; -import gov.loc.repository.bagit.hash.SupportedAlgorithm; +import nl.knaw.dans.bagit.hash.SupportedAlgorithm; /** * A manifest is a list of files and their corresponding checksum with the algorithm used to generate that checksum diff --git a/src/main/java/gov/loc/repository/bagit/domain/Metadata.java b/src/main/java/nl/knaw/dans/bagit/domain/Metadata.java similarity index 83% rename from src/main/java/gov/loc/repository/bagit/domain/Metadata.java rename to src/main/java/nl/knaw/dans/bagit/domain/Metadata.java index bcafa9c72..2950fbcec 100644 --- a/src/main/java/gov/loc/repository/bagit/domain/Metadata.java +++ b/src/main/java/nl/knaw/dans/bagit/domain/Metadata.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; @@ -63,7 +78,7 @@ public List get(final String key){ * @param key the label * @param value the value of the label * - * @return true (as specified by {@link Collection#add}) + * @return true (as specified by {@link Collection#add}) */ public boolean add(final String key, final String value){ if(PAYLOAD_OXUM.equalsIgnoreCase(key)){ @@ -121,7 +136,7 @@ public void addAll(final List> data){ * payload oxum is a special case where it makes no sense to have multiple values so instead of just appending we upsert (insert or update) * @param payloadOxumValue the value payload-oxum should be set to * - * @return true (as specified by {@link Collection#add}) + * @return true (as specified by {@link Collection#add}) */ public boolean upsertPayloadOxum(final String payloadOxumValue){ map.remove(PAYLOAD_OXUM.toUpperCase()); diff --git a/src/main/java/gov/loc/repository/bagit/domain/Version.java b/src/main/java/nl/knaw/dans/bagit/domain/Version.java similarity index 73% rename from src/main/java/gov/loc/repository/bagit/domain/Version.java rename to src/main/java/nl/knaw/dans/bagit/domain/Version.java index b332ddc25..f890cb9a6 100644 --- a/src/main/java/gov/loc/repository/bagit/domain/Version.java +++ b/src/main/java/nl/knaw/dans/bagit/domain/Version.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.util.Objects; diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/CorruptChecksumException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/CorruptChecksumException.java new file mode 100644 index 000000000..e0c767254 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/CorruptChecksumException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +import java.nio.file.Path; + +import org.slf4j.helpers.MessageFormatter; + +/** + * Class to represent an error when the calculated checksum is different than the manifest specified checksum. + */ +public class CorruptChecksumException extends Exception { + private static final long serialVersionUID = 1L; + + public CorruptChecksumException(final String message, final Path path, final String algorithm, final String hash, final String computedHash){ + super(MessageFormatter.arrayFormat(message, new Object[]{path, algorithm, hash, computedHash}).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/FileNotInManifestException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/FileNotInManifestException.java new file mode 100644 index 000000000..2825c0479 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/FileNotInManifestException.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +import java.io.IOException; + +/** + * Class to represent an error when a file is found in the payload directory but not in any manifest. + * Opposite to {@link FileNotInPayloadDirectoryException} + */ +public class FileNotInManifestException extends IOException { + private static final long serialVersionUID = 1L; + + public FileNotInManifestException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/FileNotInPayloadDirectoryException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/FileNotInPayloadDirectoryException.java new file mode 100644 index 000000000..0e9a6a26e --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/FileNotInPayloadDirectoryException.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * Class to represent an error when a file is not in the payload directory but is listed in a manifest. + * Opposite to {@link FileNotInManifestException} + */ +public class FileNotInPayloadDirectoryException extends Exception { + private static final long serialVersionUID = 1L; + + public FileNotInPayloadDirectoryException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidBagMetadataException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidBagMetadataException.java new file mode 100644 index 000000000..1ec436d1d --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidBagMetadataException.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * Class to represent an error when the bag metadata file does not conform to the bagit spec, + * namely:
+ * <KEY>:<VALUE> + *
or + * 
<KEY>:<VALUE>
+ *    <VALUE CONTINUED>
+ */ +public class InvalidBagMetadataException extends InvalidBagitFileFormatException { + private static final long serialVersionUID = 1L; + + public InvalidBagMetadataException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidBagitFileFormatException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidBagitFileFormatException.java new file mode 100644 index 000000000..c9df83dd8 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidBagitFileFormatException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * Class to represent an error when a specific bag file does not conform to its bagit specfication format + */ +public class InvalidBagitFileFormatException extends Exception { + private static final long serialVersionUID = 1L; + + public InvalidBagitFileFormatException(final String message){ + super(message); + } + + public InvalidBagitFileFormatException(final String message, final Exception e){ + super(message, e); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidPayloadOxumException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidPayloadOxumException.java new file mode 100644 index 000000000..686843195 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/InvalidPayloadOxumException.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * Class to represent an error when the calculated total bytes or number of files for + * the payload-oxum is different than the supplied values. + */ +public class InvalidPayloadOxumException extends Exception { + private static final long serialVersionUID = 1L; + + public InvalidPayloadOxumException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/MaliciousPathException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/MaliciousPathException.java new file mode 100644 index 000000000..720de9866 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/MaliciousPathException.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * Class to represent an error when the path in a manifest or fetch file has been crafted to point to a file or + * directory outside the bag. Most likely to try and overwrite an important system file. + */ +public class MaliciousPathException extends Exception { + private static final long serialVersionUID = 1L; + + public MaliciousPathException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/MissingBagitFileException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/MissingBagitFileException.java new file mode 100644 index 000000000..d4808e24c --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/MissingBagitFileException.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * The bagit.txt file is a required file. This class represents the error if that file is not present. + */ +public class MissingBagitFileException extends Exception { + private static final long serialVersionUID = 1L; + + public MissingBagitFileException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/MissingPayloadDirectoryException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/MissingPayloadDirectoryException.java new file mode 100644 index 000000000..4b4090e18 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/MissingPayloadDirectoryException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +import java.nio.file.Path; + +import org.slf4j.helpers.MessageFormatter; + +/** + * The payload directory is a required file. This class represents the error if it is not found. + */ +public class MissingPayloadDirectoryException extends Exception { + private static final long serialVersionUID = 1L; + + public MissingPayloadDirectoryException(final String message, final Path path){ + super(MessageFormatter.format(message, path).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/MissingPayloadManifestException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/MissingPayloadManifestException.java new file mode 100644 index 000000000..03d04ef8e --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/MissingPayloadManifestException.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * A bagit bag needs at least one payload manifest. This class represents the error if at least one payload manifest isn't found. + */ +public class MissingPayloadManifestException extends Exception { + private static final long serialVersionUID = 1L; + + public MissingPayloadManifestException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/PayloadOxumDoesNotExistException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/PayloadOxumDoesNotExistException.java new file mode 100644 index 000000000..e2b34be4e --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/PayloadOxumDoesNotExistException.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +import nl.knaw.dans.bagit.domain.Bag; + +/** + * The {@link Bag} object should contain the Payload-Oxum metatdata key value pair, + * this class represents the error when trying to calculate the payload-oxum and it doesn't exist on the bag object. + */ +public class PayloadOxumDoesNotExistException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public PayloadOxumDoesNotExistException(final String message){ + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/UnparsableVersionException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/UnparsableVersionException.java new file mode 100644 index 000000000..cc9f588ec --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/UnparsableVersionException.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +import org.slf4j.helpers.MessageFormatter; + +/** + * If the version string in the bagit.txt file was not in the form <MAJOR>.<MINOR> + */ +public class UnparsableVersionException extends Exception { + private static final long serialVersionUID = 1L; + + public UnparsableVersionException(final String message, final String version){ + super(MessageFormatter.format(message, version).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/UnsupportedAlgorithmException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/UnsupportedAlgorithmException.java new file mode 100644 index 000000000..ea3865723 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/UnsupportedAlgorithmException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +import java.security.MessageDigest; + +import org.slf4j.helpers.MessageFormatter; + +/** + * When the bag uses an checksum algorithm that is not supported by {@link MessageDigest}. + */ +public class UnsupportedAlgorithmException extends Exception { + private static final long serialVersionUID = 1L; + + public UnsupportedAlgorithmException(final String message, final String bagitAlgorithmName, final Throwable cause) { + super(MessageFormatter.format(message, bagitAlgorithmName).getMessage(), cause); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/VerificationException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/VerificationException.java new file mode 100644 index 000000000..52df6e1ce --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/VerificationException.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions; + +/** + * Class to represent an generic exception that happened during verification. + */ +public class VerificationException extends Exception { + private static final long serialVersionUID = 1L; + + public VerificationException(final Exception exception){ + super(exception); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/BagitVersionIsNotAcceptableException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/BagitVersionIsNotAcceptableException.java new file mode 100644 index 000000000..7e6e2ea39 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/BagitVersionIsNotAcceptableException.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions.conformance; + +import java.util.List; + +import nl.knaw.dans.bagit.domain.Version; +import org.slf4j.helpers.MessageFormatter; + +/** + * Class to represent when the bag's version is not in the acceptable list of versions + */ +public class BagitVersionIsNotAcceptableException extends Exception { +private static final long serialVersionUID = 1L; + + public BagitVersionIsNotAcceptableException(final String message, final Version version, final List acceptableVersions) { + super(MessageFormatter.format(message, version, acceptableVersions).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/FetchFileNotAllowedException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/FetchFileNotAllowedException.java new file mode 100644 index 000000000..c73755afe --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/FetchFileNotAllowedException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions.conformance; + +import java.nio.file.Path; + +import org.slf4j.helpers.MessageFormatter; + +/** + * Class to represent when a fetch file is found in a bag but is not allowed according to the bagit profile + */ +public class FetchFileNotAllowedException extends Exception { +private static final long serialVersionUID = 1L; + + public FetchFileNotAllowedException(final String message, final Path rootDir) { + super(MessageFormatter.format(message, rootDir).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/MetatdataValueIsNotAcceptableException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/MetatdataValueIsNotAcceptableException.java new file mode 100644 index 000000000..cdb2847c7 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/MetatdataValueIsNotAcceptableException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions.conformance; + +import java.util.List; + +import org.slf4j.helpers.MessageFormatter; + +/** + * Class to represent when a metadata's value is not in the acceptable list of values + */ +public class MetatdataValueIsNotAcceptableException extends Exception { +private static final long serialVersionUID = 1L; + + public MetatdataValueIsNotAcceptableException(final String message, final String metadataKey, final List acceptableValues, final String actualValue) { + super(MessageFormatter.arrayFormat(message, new Object[]{metadataKey, acceptableValues, actualValue}).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/MetatdataValueIsNotRepeatableException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/MetatdataValueIsNotRepeatableException.java new file mode 100644 index 000000000..e68908850 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/MetatdataValueIsNotRepeatableException.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions.conformance; + +import org.slf4j.helpers.MessageFormatter; + +/** + * Class to represent when a metadata's value is not to be repeated + */ +public class MetatdataValueIsNotRepeatableException extends Exception { +private static final long serialVersionUID = 1L; + + public MetatdataValueIsNotRepeatableException(final String message, final String metadataKey) { + super(MessageFormatter.format(message, metadataKey).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredManifestNotPresentException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredManifestNotPresentException.java new file mode 100644 index 000000000..28fb7b7cf --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredManifestNotPresentException.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions.conformance; + +/** + * Class to represent when a specific manifest type is not found, such as md5, sha1, etc (payload or tag) + */ +public class RequiredManifestNotPresentException extends Exception { +private static final long serialVersionUID = 1L; + + public RequiredManifestNotPresentException(final String message) { + super(message); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredMetadataFieldNotPresentException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredMetadataFieldNotPresentException.java new file mode 100644 index 000000000..7b6f5aa51 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredMetadataFieldNotPresentException.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions.conformance; + +import org.slf4j.helpers.MessageFormatter; + +/** + * Class to represent when a specific metadata field is not found + */ +public class RequiredMetadataFieldNotPresentException extends Exception { +private static final long serialVersionUID = 1L; + + public RequiredMetadataFieldNotPresentException(final String message, final String bagInfoEntryRequirementKey) { + super(MessageFormatter.format(message, bagInfoEntryRequirementKey).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredTagFileNotPresentException.java b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredTagFileNotPresentException.java new file mode 100644 index 000000000..fd72310c3 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/exceptions/conformance/RequiredTagFileNotPresentException.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.exceptions.conformance; + +import org.slf4j.helpers.MessageFormatter; + +/** + * Class to represent when a specific tag file is not found + */ +public class RequiredTagFileNotPresentException extends Exception { +private static final long serialVersionUID = 1L; + + public RequiredTagFileNotPresentException(final String message, final String requiredTagFilePath) { + super(MessageFormatter.format(message, requiredTagFilePath).getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/hash/BagitAlgorithmNameToSupportedAlgorithmMapping.java b/src/main/java/nl/knaw/dans/bagit/hash/BagitAlgorithmNameToSupportedAlgorithmMapping.java new file mode 100644 index 000000000..2772fc929 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/hash/BagitAlgorithmNameToSupportedAlgorithmMapping.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.hash; + +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; + +/** + * Implement this interface if you need to be able to use other algorithms than the {@link StandardSupportedAlgorithms} + */ +public interface BagitAlgorithmNameToSupportedAlgorithmMapping { + SupportedAlgorithm getSupportedAlgorithm(String bagitAlgorithmName) throws UnsupportedAlgorithmException; +} diff --git a/src/main/java/nl/knaw/dans/bagit/hash/Hasher.java b/src/main/java/nl/knaw/dans/bagit/hash/Hasher.java new file mode 100644 index 000000000..a17e0b2ba --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/hash/Hasher.java @@ -0,0 +1,623 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.hash; + +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.domain.Manifest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.ProtocolException; +import java.net.URL; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Collections; +import java.util.Formatter; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.ResourceBundle; + +/** + * Convenience class for generating a HEX formatted string of the checksum hash. + */ +public final class Hasher { + private static final Logger logger = LoggerFactory.getLogger(Hasher.class); + private static final int _64_KB = 1024 * 64; + private static final int CHUNK_SIZE = _64_KB; + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); + + public static final String CHUNK_SIZE_PROP = "nl.knaw.dans.bagit.hash.chunkSize"; + public static final String MAX_RETRIES_PROP = "nl.knaw.dans.bagit.hash.maxRetries"; + public static final String RETRY_SLEEP_MS_PROP = "nl.knaw.dans.bagit.hash.retrySleepMs"; + public static final String MAX_REDIRECTS_PROP = "nl.knaw.dans.bagit.hash.maxRedirects"; + public static final String FALL_BACK_TO_FULL_STREAM_ON_RANGE_FAIL_PROP = "nl.knaw.dans.bagit.hash.fallBackToFullStreamOnRangeFail"; + + private static final int DEFAULT_CHUNK_SIZE = 1024 * 1024 * 128; // 128 MB + private static final int DEFAULT_MAX_RETRIES = 5; + private static final int DEFAULT_RETRY_SLEEP_MS = 5000; + private static final int DEFAULT_MAX_REDIRECTS = 20; + private static final boolean DEFAULT_FALL_BACK_TO_FULL_STREAM_ON_RANGE_FAIL = false; + + private Hasher() { + //intentionally left empty + } + + /** + * Create a HEX formatted string checksum hash of the file + * + * @param path the {@link Path} (file) to hash + * @param messageDigest the {@link MessageDigest} object representing the hashing algorithm + * @return the hash as a hex formated string + * @throws IOException if there is a problem reading the file + */ + public static String hash(final Path path, final MessageDigest messageDigest) throws IOException { + updateMessageDigests(path, Collections.singletonList(messageDigest)); + + return formatMessageDigest(messageDigest); + } + + /** + * Create a HEX formatted string checksum hash of the data from the URL + * + * @param url the {@link URL} to hash + * @param messageDigest the {@link MessageDigest} object representing the hashing algorithm + * @return the hash as a hex-formatted string + * @throws IOException if there is a problem reading from the URL + */ + public static String hash(final URL url, final MessageDigest messageDigest) throws IOException { + return hash(url, messageDigest, null); + } + + /** + * Create a HEX-formatted string checksum hash of the data from the {@link FetchItem} + * + * @param item the {@link FetchItem} to hash + * @param messageDigest the {@link MessageDigest} object representing the hashing algorithm + * @param extraHeaders optional extra headers to send with the request + * @return the hash as a hex-formatted string + * @throws IOException if there is a problem reading from the URL + */ + public static String hash(final FetchItem item, final MessageDigest messageDigest, final Map extraHeaders) throws IOException { + return hash(item, messageDigest, extraHeaders, HashOptions.systemProperties()); + } + + /** + * Create a HEX-formatted string checksum hash of the data from the {@link FetchItem} + * + * @param item the {@link FetchItem} to hash + * @param messageDigest the {@link MessageDigest} object representing the hashing algorithm + * @param extraHeaders optional extra headers to send with the request + * @param hashOptions optional settings for ranged HTTP requests + * @return the hash as a hex-formatted string + * @throws IOException if there is a problem reading from the URL + */ + public static String hash(final FetchItem item, final MessageDigest messageDigest, final Map extraHeaders, final HashOptions hashOptions) throws IOException { + long totalSize = (item.length != null && item.length >= 0) ? item.length : -1; + URL currentUrl = item.url; + Map currentHeaders = extraHeaders; + + if (!currentUrl.getProtocol().startsWith("http")) { + return hashFullStream(currentUrl, messageDigest, currentHeaders, hashOptions); + } + + HashOptions effectiveOptions = hashOptions == null ? HashOptions.systemProperties() : hashOptions; + long chunkSize = effectiveOptions.getChunkSize(); + int maxRetries = effectiveOptions.getMaxRetries(); + int retrySleepMs = effectiveOptions.getRetrySleepMs(); + int maxRedirects = effectiveOptions.getMaxRedirects(); + int redirectCount = 0; + + long offset = 0; + while (totalSize < 0 || offset < totalSize) { + long end = (totalSize > 0) ? Math.min(offset + chunkSize - 1, totalSize - 1) : offset + chunkSize - 1; + String range = "bytes=" + offset + "-" + end; + + final URL finalUrl = currentUrl; + final Map finalHeaders = currentHeaders; + final long finalOffset = offset; + final long finalTotalSize = totalSize; + + try { + ChunkResult result = executeWithRetry(() -> { + HttpURLConnection conn = openRangedConnection(finalUrl, range, finalHeaders); + int code = conn.getResponseCode(); + + // Manual redirect following + if (code >= 300 && code < 400) { + return ChunkResult.redirect(conn.getHeaderField("Location")); + } + + if (code == 206) { + return handlePartialContent(conn, messageDigest, finalTotalSize, range); + } + else if (code == 200) { + if (finalOffset > 0) { + throw new ProtocolException("Server returned 200 OK for range request " + range + " from " + finalUrl); + } + logger.info("Server returned 200 OK for first range request (probably range requests are not supported); downloading full stream from {}", finalUrl); + messageDigest.reset(); // Reset in case we got here after a retry + try (InputStream is = conn.getInputStream()) { + long totalRead = updateDigestFromStream(is, messageDigest); + if (finalTotalSize >= 0 && totalRead != finalTotalSize) { + throw new IOException("Expected to read " + finalTotalSize + " bytes but read " + totalRead + " bytes from " + finalUrl); + } + } + return ChunkResult.fullStream(formatMessageDigest(messageDigest)); + } + else { + throw new IOException("Unexpected response code " + code + " for " + finalUrl); + } + }, "Error fetching range " + range + " from " + currentUrl, maxRetries, retrySleepMs); + + logger.debug("Processing chunk result for range {} from {}", range, currentUrl); + + if (result.type == ChunkResultType.FULL_STREAM_SUCCESS) { + redirectCount = 0; + logger.debug("Successfully processed full stream for range {} from {}", range, currentUrl); + return result.hash; + } + else if (result.type == ChunkResultType.REDIRECT) { + if (++redirectCount > maxRedirects) { + throw new ProtocolException("Too many redirects"); + } + URL nextUrl = new URL(currentUrl, result.location); + if (!currentUrl.getAuthority().equals(nextUrl.getAuthority()) || !currentUrl.getProtocol().equals(nextUrl.getProtocol())) { + currentHeaders = null; + } + currentUrl = nextUrl; + logger.debug("Redirected to {}, currentHeaders stripped: {}", currentUrl, (currentHeaders == null)); + // Skip offset update and retry the current chunk with new URL + } + else if (result.type == ChunkResultType.SUCCESS) { + redirectCount = 0; + currentUrl = item.url; + currentHeaders = extraHeaders; + offset += result.bytesRead; + if (totalSize < 0 && result.totalSize > 0) { + totalSize = result.totalSize; + } + logger.debug("Successfully processed chunk for range {} from {}", range, currentUrl); + logger.debug("Read {} of {}{}", offset, totalSize > 0 ? totalSize : "Unknown", totalSize > 0 ? " (" + (offset * 100L / totalSize) + "%)" : ""); + } + } + catch (ProtocolException e) { + throw e; + } + catch (IOException e) { + if (effectiveOptions.isFallBackToFullStreamOnRangeFail()) { + logger.info("Falling back to full stream for {} after failed range requests", currentUrl); + messageDigest.reset(); + return hashFullStream(currentUrl, messageDigest, currentHeaders, hashOptions); + } + else { + logger.error("Failed range requests for {}, and fallback to full stream is disabled", currentUrl); + throw e; + } + } + } + + return formatMessageDigest(messageDigest); + } + + private static String hashFullStream(final URL url, final MessageDigest messageDigest, final Map extraHeaders, final HashOptions hashOptions) throws IOException { + HashOptions effectiveOptions = hashOptions == null ? HashOptions.systemProperties() : hashOptions; + int maxRetries = effectiveOptions.getMaxRetries(); + int retrySleepMs = effectiveOptions.getRetrySleepMs(); + int maxRedirects = effectiveOptions.getMaxRedirects(); + + for (int attempt = 0; attempt < maxRetries; attempt++) { + URL currentUrl = url; + Map currentHeaders = extraHeaders; + try { + int redirectCount = 0; + while (true) { + URLConnection conn = currentUrl.openConnection(); + if (conn instanceof HttpURLConnection httpConn) { + httpConn.setInstanceFollowRedirects(false); + if (currentHeaders != null) { + for (Entry entry : currentHeaders.entrySet()) { + httpConn.setRequestProperty(entry.getKey(), entry.getValue()); + } + } + int code = httpConn.getResponseCode(); + if (code >= 300 && code < 400) { + if (++redirectCount > maxRedirects) { + throw new ProtocolException("Too many redirects"); + } + String location = httpConn.getHeaderField("Location"); + URL nextUrl = new URL(currentUrl, location); + if (!currentUrl.getAuthority().equals(nextUrl.getAuthority()) || !currentUrl.getProtocol().equals(nextUrl.getProtocol())) { + currentHeaders = null; + } + currentUrl = nextUrl; + continue; + } + if (code != 200) { + throw new IOException("Unexpected response code " + code + " for " + currentUrl); + } + } + try (final InputStream is = conn.getInputStream()) { + updateDigestFromStream(is, messageDigest); + } + break; + } + return formatMessageDigest(messageDigest); + } + catch (ProtocolException e) { + throw e; + } + catch (IOException e) { + logger.warn("Error fetching full stream from {} (attempt {}/{}): {}", url, attempt + 1, maxRetries, e.getMessage()); + messageDigest.reset(); + if (attempt < maxRetries - 1) { + try { + Thread.sleep(retrySleepMs); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during retry sleep", ie); + } + } + else { + throw e; + } + } + } + throw new IOException("Max retries exceeded"); + } + + /** + * Create a HEX formatted string checksum hash of the data from the URL + * + * @param url the {@link URL} to hash + * @param messageDigest the {@link MessageDigest} object representing the hashing algorithm + * @param extraHeaders optional extra headers to send with the request + * @return the hash as a hex formatted string + * @throws IOException if there is a problem reading from the URL + */ + public static String hash(final URL url, final MessageDigest messageDigest, final Map extraHeaders) throws IOException { + return hash(new FetchItem(url, -1L, null), messageDigest, extraHeaders); + } + + /** + * Create a HEX formatted string checksum hash of the data from the URL + * + * @param url the {@link URL} to hash + * @param messageDigest the {@link MessageDigest} object representing the hashing algorithm + * @param extraHeaders optional extra headers to send with the request + * @param hashOptions optional settings for ranged HTTP requests + * @return the hash as a hex formatted string + * @throws IOException if there is a problem reading from the URL + */ + public static String hash(final URL url, final MessageDigest messageDigest, final Map extraHeaders, final HashOptions hashOptions) throws IOException { + return hash(new FetchItem(url, -1L, null), messageDigest, extraHeaders, hashOptions); + } + + private static ChunkResult handlePartialContent(HttpURLConnection conn, MessageDigest messageDigest, long currentTotalSize, String range) throws IOException { + long totalSize = currentTotalSize; + if (totalSize < 0) { + String contentRange = conn.getHeaderField("Content-Range"); + if (contentRange != null && contentRange.contains("/")) { + try { + totalSize = Long.parseLong(contentRange.substring(contentRange.lastIndexOf("/") + 1)); + } + catch (NumberFormatException e) { + logger.warn("Could not parse Content-Range: {}", contentRange); + } + } + } + try (InputStream is = conn.getInputStream()) { + byte[] bytes = is.readAllBytes(); + if (bytes.length == 0) { + // N.B. getting the range from the connection at this point may throw an exception, so we use the range we already have. + throw new IOException("Received empty response for range request " + range + " from " + conn.getURL()); + } + messageDigest.update(bytes); + logger.debug("Updated message digest with range {}", range); + return ChunkResult.success(bytes.length, totalSize); + } + } + + private static ChunkResult executeWithRetry(RetryableOperation operation, String description, int maxRetries, int retrySleepMs) throws IOException { + for (int attempt = 0; attempt < maxRetries; attempt++) { + try { + return operation.execute(); + } + catch (ProtocolException e) { + throw e; + } + catch (IOException e) { + logger.warn("{} (attempt {}/{}): {}", description, attempt + 1, maxRetries, e.getMessage()); + if (attempt < maxRetries - 1) { + try { + Thread.sleep(retrySleepMs); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during retry sleep", ie); + } + } + else { + throw e; + } + } + } + throw new IOException("Max retries exceeded"); + } + + @FunctionalInterface + private interface RetryableOperation { + T execute() throws IOException; + } + + private enum ChunkResultType { + SUCCESS, REDIRECT, FULL_STREAM_SUCCESS + } + + /** + * Represents the result of a chunk operation when processing data for hashing. A chunk operation can have various results, such as a successful read, a redirection to another location, or + * successful processing of a complete stream. + */ + private static class ChunkResult { + final ChunkResultType type; + final long bytesRead; + final long totalSize; + final String location; + final String hash; + + private ChunkResult(ChunkResultType type, long bytesRead, long totalSize, String location, String hash) { + this.type = type; + this.bytesRead = bytesRead; + this.totalSize = totalSize; + this.location = location; + this.hash = hash; + } + + static ChunkResult success(long bytesRead, long totalSize) { + return new ChunkResult(ChunkResultType.SUCCESS, bytesRead, totalSize, null, null); + } + + static ChunkResult redirect(String location) { + return new ChunkResult(ChunkResultType.REDIRECT, 0, -1, location, null); + } + + static ChunkResult fullStream(String hash) { + return new ChunkResult(ChunkResultType.FULL_STREAM_SUCCESS, 0, -1, null, hash); + } + } + + /** + * Opens a HttpURLConnection for the given range. + */ + private static HttpURLConnection openRangedConnection(URL url, String range, final Map extraHeaders) throws IOException { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + if (extraHeaders != null) { + for (Entry entry : extraHeaders.entrySet()) { + conn.setRequestProperty(entry.getKey(), entry.getValue()); + } + } + conn.setInstanceFollowRedirects(false); + if (range != null) { + conn.setRequestProperty("Range", range); + } + return conn; + } + + /** + * Reads from the InputStream and updates the MessageDigest. Returns the number of bytes read. + */ + private static long updateDigestFromStream(InputStream is, MessageDigest messageDigest) throws IOException { + byte[] buffer = new byte[CHUNK_SIZE]; + long totalRead = 0; + int read = is.read(buffer); + while (read != -1) { + messageDigest.update(buffer, 0, read); + totalRead += read; + read = is.read(buffer); + } + return totalRead; + } + + /** + * Update the Manifests with the file's hash + * + * @param path the {@link Path} (file) to hash + * @param manifestToMessageDigestMap the map between {@link Manifest} and {@link MessageDigest} + * @throws IOException if there is a problem reading the file + */ + public static void hash(final Path path, final Map manifestToMessageDigestMap) throws IOException { + updateMessageDigests(path, manifestToMessageDigestMap.values()); + addMessageDigestHashToManifest(path, manifestToMessageDigestMap); + } + + static void updateMessageDigests(final Path path, final Collection messageDigests) throws IOException { + try (final InputStream is = new BufferedInputStream(Files.newInputStream(path, StandardOpenOption.READ))) { + final byte[] buffer = new byte[CHUNK_SIZE]; + int read = is.read(buffer); + + while (read != -1) { + for (final MessageDigest messageDigest : messageDigests) { + messageDigest.update(buffer, 0, read); + } + read = is.read(buffer); + } + } + } + + private static void addMessageDigestHashToManifest(final Path path, final Map manifestToMessageDigestMap) { + for (final Entry entry : manifestToMessageDigestMap.entrySet()) { + final String hash = formatMessageDigest(entry.getValue()); + logger.debug(messages.getString("adding_checksum"), path, hash); + entry.getKey().getFileToChecksumMap().put(path, hash); + } + } + + //Convert the byte to hex format + private static String formatMessageDigest(final MessageDigest messageDigest) { + try (final Formatter formatter = new Formatter()) { + for (final byte b : messageDigest.digest()) { + formatter.format("%02x", b); + } + + return formatter.toString(); + } + } + + /** + * create a mapping between {@link Manifest} and {@link MessageDigest} for each each supplied {@link SupportedAlgorithm} + * + * @param algorithms the {@link SupportedAlgorithm} that you which to map to {@link MessageDigest} + * @return mapping between {@link Manifest} and {@link MessageDigest} + * @throws NoSuchAlgorithmException if {@link MessageDigest} doesn't support the algorithm + */ + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + public static Map createManifestToMessageDigestMap(final Collection algorithms) throws NoSuchAlgorithmException { + final Map map = new HashMap<>(); + + for (final SupportedAlgorithm algorithm : algorithms) { + final MessageDigest messageDigest = MessageDigest.getInstance(algorithm.getMessageDigestName()); + final Manifest manifest = new Manifest(algorithm); + map.put(manifest, messageDigest); + } + + return map; + } + + /** + * Represents configuration options for hashing operations. This class is immutable and provides various configurable parameters such as chunk size, retry behavior, and redirection limits. + */ + public static final class HashOptions { + private final int chunkSize; + private final int maxRetries; + private final int retrySleepMs; + private final int maxRedirects; + private final boolean fallBackToFullStreamOnRangeFail; + + /** + * Constructs an instance of {@code HashOptions} with specific configuration settings for chunk size, retry behavior, and redirection limits. + * + * @param chunkSize the size of data chunks in bytes; must be greater than 0 + * @param maxRetries the maximum number of retry attempts; must be greater than 0 + * @param retrySleepMs the time in milliseconds to sleep between retries + * @param maxRedirects the maximum number of redirects allowed; must be greater than or equal to 0 + * @throws IllegalArgumentException if any of the provided values are invalid (e.g., negative or zero where not allowed) + */ + public HashOptions(final int chunkSize, final int maxRetries, final int retrySleepMs, final int maxRedirects, boolean fallBackToFullStreamOnRangeFail) { + if (chunkSize <= 0) { + throw new IllegalArgumentException("chunkSize must be greater than 0"); + } + if (maxRetries <= 0) { + throw new IllegalArgumentException("maxRetries must be greater than 0"); + } + if (maxRedirects < 0) { + throw new IllegalArgumentException("maxRedirects must be greater than or equal to 0"); + } + this.chunkSize = chunkSize; + this.maxRetries = maxRetries; + this.retrySleepMs = retrySleepMs; + this.maxRedirects = maxRedirects; + this.fallBackToFullStreamOnRangeFail = fallBackToFullStreamOnRangeFail; + } + + /** + * Creates an instance of {@code HashOptions} using system properties to determine the configuration values for chunk size, maximum retries, retry sleep time, and maximum redirects. Defaults + * are used if the respective system properties are not set. + * + * The following system properties are used: - {@code CHUNK_SIZE_PROP}: Configures the chunk size in bytes (default: {@code DEFAULT_CHUNK_SIZE}). - {@code MAX_RETRIES_PROP}: Configures the + * maximum number of retry attempts (default: {@code DEFAULT_MAX_RETRIES}). - {@code RETRY_SLEEP_MS_PROP}: Configures the sleep time in milliseconds between retries (default: + * {@code DEFAULT_RETRY_SLEEP_MS}). - {@code MAX_REDIRECTS_PROP}: Configures the maximum allowable redirects (default: {@code DEFAULT_MAX_REDIRECTS}). + * + * @return a {@code HashOptions} instance populated with configuration values derived from system properties or their default values. + * @throws IllegalArgumentException if any of the retrieved values are invalid (e.g., negative or zero where not allowed) + */ + public static HashOptions systemProperties() { + return new HashOptions( + Integer.getInteger(CHUNK_SIZE_PROP, DEFAULT_CHUNK_SIZE), + Integer.getInteger(MAX_RETRIES_PROP, DEFAULT_MAX_RETRIES), + Integer.getInteger(RETRY_SLEEP_MS_PROP, DEFAULT_RETRY_SLEEP_MS), + Integer.getInteger(MAX_REDIRECTS_PROP, DEFAULT_MAX_REDIRECTS), + Boolean.parseBoolean(System.getProperty(FALL_BACK_TO_FULL_STREAM_ON_RANGE_FAIL_PROP, String.valueOf(DEFAULT_FALL_BACK_TO_FULL_STREAM_ON_RANGE_FAIL))) + ); + } + + /** + * Returns a new {@code HashOptions} instance with the specified overrides for configuration parameters. If a parameter is {@code null}, the existing value from the current {@code HashOptions} + * instance is used. + * + * @param chunkSize the size of data chunks in bytes; if {@code null}, the existing chunk size is retained + * @param maxRetries the maximum number of retry attempts; if {@code null}, the existing max retries value is retained + * @param retrySleepMs the time in milliseconds to sleep between retries; if {@code null}, the existing retry sleep time is retained + * @param maxRedirects the maximum number of redirects allowed; if {@code null}, the existing max redirects value is retained + * @return a new {@code HashOptions} instance with the specified values or the existing values if overrides are {@code null} + * @throws IllegalArgumentException if any of the provided values are invalid (e.g., negative or zero where not allowed) + */ + public HashOptions withOverrides(final Integer chunkSize, final Integer maxRetries, final Integer retrySleepMs, final Integer maxRedirects) { + return withOverrides(chunkSize, maxRetries, retrySleepMs, maxRedirects, null); + } + + /** + * Returns a new {@code HashOptions} instance with the specified overrides for configuration parameters. If a parameter is {@code null}, the existing value from the current {@code HashOptions} + * instance is used. + * + * @param chunkSize the size of data chunks in bytes; if {@code null}, the existing chunk size is retained + * @param maxRetries the maximum number of retry attempts; if {@code null}, the existing max retries value is retained + * @param retrySleepMs the time in milliseconds to sleep between retries; if {@code null}, the existing retry sleep time is retained + * @param maxRedirects the maximum number of redirects allowed; if {@code null}, the existing max redirects value is retained + * @param fallBackToFullStreamOnRangeFail whether to fall back to full stream; if {@code null}, the existing value is retained + * @return a new {@code HashOptions} instance with the specified values or the existing values if overrides are {@code null} + * @throws IllegalArgumentException if any of the provided values are invalid (e.g., negative or zero where not allowed) + */ + public HashOptions withOverrides(final Integer chunkSize, final Integer maxRetries, final Integer retrySleepMs, final Integer maxRedirects, final Boolean fallBackToFullStreamOnRangeFail) { + return new HashOptions( + chunkSize == null ? this.chunkSize : chunkSize, + maxRetries == null ? this.maxRetries : maxRetries, + retrySleepMs == null ? this.retrySleepMs : retrySleepMs, + maxRedirects == null ? this.maxRedirects : maxRedirects, + fallBackToFullStreamOnRangeFail == null ? this.fallBackToFullStreamOnRangeFail : fallBackToFullStreamOnRangeFail + ); + } + + public int getChunkSize() { + return chunkSize; + } + + public int getMaxRetries() { + return maxRetries; + } + + public int getRetrySleepMs() { + return retrySleepMs; + } + + public int getMaxRedirects() { + return maxRedirects; + } + + public boolean isFallBackToFullStreamOnRangeFail() { + return fallBackToFullStreamOnRangeFail; + } + } +} diff --git a/src/main/java/gov/loc/repository/bagit/hash/StandardBagitAlgorithmNameToSupportedAlgorithmMapping.java b/src/main/java/nl/knaw/dans/bagit/hash/StandardBagitAlgorithmNameToSupportedAlgorithmMapping.java similarity index 51% rename from src/main/java/gov/loc/repository/bagit/hash/StandardBagitAlgorithmNameToSupportedAlgorithmMapping.java rename to src/main/java/nl/knaw/dans/bagit/hash/StandardBagitAlgorithmNameToSupportedAlgorithmMapping.java index d35e71cb2..c3e5aa63b 100644 --- a/src/main/java/gov/loc/repository/bagit/hash/StandardBagitAlgorithmNameToSupportedAlgorithmMapping.java +++ b/src/main/java/nl/knaw/dans/bagit/hash/StandardBagitAlgorithmNameToSupportedAlgorithmMapping.java @@ -1,9 +1,24 @@ -package gov.loc.repository.bagit.hash; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.hash; import java.util.Locale; import java.util.ResourceBundle; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; /** * Provides a mapping between bagit algorithm names and {@link SupportedAlgorithm} diff --git a/src/main/java/nl/knaw/dans/bagit/hash/StandardSupportedAlgorithms.java b/src/main/java/nl/knaw/dans/bagit/hash/StandardSupportedAlgorithms.java new file mode 100644 index 000000000..b7ee4647d --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/hash/StandardSupportedAlgorithms.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.hash; + +/** + * The standard algorithms that are supported "out of the box" in bagit + */ +public enum StandardSupportedAlgorithms implements SupportedAlgorithm{ + MD5("MD5"), + SHA1("SHA-1"), + SHA224("SHA-224"), + SHA256("SHA-256"), + SHA512("SHA-512"); + + private final String messageDigestName; + + private StandardSupportedAlgorithms(final String messageDigestName){ + this.messageDigestName = messageDigestName; + } + + @Override + public String getMessageDigestName() { + return messageDigestName; + } + + @SuppressWarnings({"PMD.UseLocaleWithCaseConversions"}) + @Override + public String getBagitName() { + return name().toLowerCase(); + } +} diff --git a/src/main/java/nl/knaw/dans/bagit/hash/SupportedAlgorithm.java b/src/main/java/nl/knaw/dans/bagit/hash/SupportedAlgorithm.java new file mode 100644 index 000000000..c6eb96e11 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/hash/SupportedAlgorithm.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.hash; + +import java.security.MessageDigest; + +/** + * Easy way to convert between bagit manifest spec and {@link MessageDigest}
+ * See {@link StandardSupportedAlgorithms} for a list of defaults + */ +public interface SupportedAlgorithm { + String getMessageDigestName(); + String getBagitName(); +} diff --git a/src/main/java/gov/loc/repository/bagit/reader/BagReader.java b/src/main/java/nl/knaw/dans/bagit/reader/BagReader.java similarity index 67% rename from src/main/java/gov/loc/repository/bagit/reader/BagReader.java rename to src/main/java/nl/knaw/dans/bagit/reader/BagReader.java index c92844d02..914906840 100644 --- a/src/main/java/gov/loc/repository/bagit/reader/BagReader.java +++ b/src/main/java/nl/knaw/dans/bagit/reader/BagReader.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.IOException; import java.nio.charset.Charset; @@ -6,15 +21,15 @@ import java.nio.file.Path; import java.util.AbstractMap.SimpleImmutableEntry; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.InvalidBagMetadataException; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.UnparsableVersionException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.exceptions.InvalidBagMetadataException; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.exceptions.UnparsableVersionException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; +import nl.knaw.dans.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; /** * Responsible for reading a bag from the filesystem. diff --git a/src/main/java/gov/loc/repository/bagit/reader/BagitTextFileReader.java b/src/main/java/nl/knaw/dans/bagit/reader/BagitTextFileReader.java similarity index 85% rename from src/main/java/gov/loc/repository/bagit/reader/BagitTextFileReader.java rename to src/main/java/nl/knaw/dans/bagit/reader/BagitTextFileReader.java index 8f8be00fa..b2fb7e882 100644 --- a/src/main/java/gov/loc/repository/bagit/reader/BagitTextFileReader.java +++ b/src/main/java/nl/knaw/dans/bagit/reader/BagitTextFileReader.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.IOException; import java.nio.charset.Charset; @@ -10,14 +25,14 @@ import java.util.AbstractMap.SimpleImmutableEntry; import java.util.Arrays; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.InvalidBagMetadataException; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.UnparsableVersionException; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.exceptions.InvalidBagMetadataException; +import nl.knaw.dans.bagit.exceptions.UnparsableVersionException; /** * This class is responsible for reading and parsing bagit.txt files from the filesystem diff --git a/src/main/java/gov/loc/repository/bagit/reader/FetchReader.java b/src/main/java/nl/knaw/dans/bagit/reader/FetchReader.java similarity index 75% rename from src/main/java/gov/loc/repository/bagit/reader/FetchReader.java rename to src/main/java/nl/knaw/dans/bagit/reader/FetchReader.java index 3e0336dda..d19cb6961 100644 --- a/src/main/java/gov/loc/repository/bagit/reader/FetchReader.java +++ b/src/main/java/nl/knaw/dans/bagit/reader/FetchReader.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.BufferedReader; import java.io.IOException; @@ -10,12 +25,12 @@ import java.util.List; import java.util.ResourceBundle; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.FetchItem; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; /** * This class is responsible for reading and parsing fetch.txt file from the filesystem diff --git a/src/main/java/gov/loc/repository/bagit/reader/KeyValueReader.java b/src/main/java/nl/knaw/dans/bagit/reader/KeyValueReader.java similarity index 79% rename from src/main/java/gov/loc/repository/bagit/reader/KeyValueReader.java rename to src/main/java/nl/knaw/dans/bagit/reader/KeyValueReader.java index c0413b2f8..f3ecf67d9 100644 --- a/src/main/java/gov/loc/repository/bagit/reader/KeyValueReader.java +++ b/src/main/java/nl/knaw/dans/bagit/reader/KeyValueReader.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.BufferedReader; import java.io.IOException; @@ -14,7 +29,7 @@ import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; -import gov.loc.repository.bagit.exceptions.InvalidBagMetadataException; +import nl.knaw.dans.bagit.exceptions.InvalidBagMetadataException; /** * Convenience class for reading key value pairs from a file @@ -68,7 +83,8 @@ public static List> readKeyValuesFromFile(f private static void mergeIndentedLine(final String line, final List> keyValues){ final SimpleImmutableEntry oldKeyValue = keyValues.remove(keyValues.size() -1); - final SimpleImmutableEntry newKeyValue = new SimpleImmutableEntry<>(oldKeyValue.getKey(), oldKeyValue.getValue() + System.lineSeparator() +line); + final String newContent = line.stripLeading(); + final SimpleImmutableEntry newKeyValue = new SimpleImmutableEntry<>(oldKeyValue.getKey(), oldKeyValue.getValue() + System.lineSeparator() + newContent); keyValues.add(newKeyValue); logger.debug(messages.getString("found_indented_line"), oldKeyValue.getKey()); diff --git a/src/main/java/gov/loc/repository/bagit/reader/ManifestReader.java b/src/main/java/nl/knaw/dans/bagit/reader/ManifestReader.java similarity index 82% rename from src/main/java/gov/loc/repository/bagit/reader/ManifestReader.java rename to src/main/java/nl/knaw/dans/bagit/reader/ManifestReader.java index fc0b86117..35bc95ae4 100644 --- a/src/main/java/gov/loc/repository/bagit/reader/ManifestReader.java +++ b/src/main/java/nl/knaw/dans/bagit/reader/ManifestReader.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.BufferedReader; import java.io.IOException; @@ -11,17 +26,17 @@ import java.util.Map; import java.util.ResourceBundle; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.hash.SupportedAlgorithm; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.hash.SupportedAlgorithm; +import nl.knaw.dans.bagit.util.PathUtils; /** * This class is responsible for reading and parsing manifest files on the filesystem diff --git a/src/main/java/gov/loc/repository/bagit/reader/MetadataReader.java b/src/main/java/nl/knaw/dans/bagit/reader/MetadataReader.java similarity index 73% rename from src/main/java/gov/loc/repository/bagit/reader/MetadataReader.java rename to src/main/java/nl/knaw/dans/bagit/reader/MetadataReader.java index adb9f9451..c62c01e4f 100644 --- a/src/main/java/gov/loc/repository/bagit/reader/MetadataReader.java +++ b/src/main/java/nl/knaw/dans/bagit/reader/MetadataReader.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.IOException; import java.nio.charset.Charset; @@ -12,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.exceptions.InvalidBagMetadataException; +import nl.knaw.dans.bagit.exceptions.InvalidBagMetadataException; /** * This class is responsible for reading and parsing bagit metadata files from the filesystem diff --git a/src/main/java/gov/loc/repository/bagit/reader/TagFileReader.java b/src/main/java/nl/knaw/dans/bagit/reader/TagFileReader.java similarity index 70% rename from src/main/java/gov/loc/repository/bagit/reader/TagFileReader.java rename to src/main/java/nl/knaw/dans/bagit/reader/TagFileReader.java index d2fceda39..c45e8b074 100644 --- a/src/main/java/gov/loc/repository/bagit/reader/TagFileReader.java +++ b/src/main/java/nl/knaw/dans/bagit/reader/TagFileReader.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.net.URI; import java.net.URISyntaxException; @@ -6,13 +21,13 @@ import java.nio.file.Paths; import java.util.ResourceBundle; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.util.PathUtils; /** * Convenience class for reading tag files from the filesystem @@ -25,7 +40,7 @@ public interface TagFileReader { /* * Create the file and check it for various things, like starting with a *, or trying to access a file outside the bag */ - static Path createFileFromManifest(final Path bagRootDir, final String path) throws MaliciousPathException, InvalidBagitFileFormatException{ + static Path createFileFromManifest(final Path bagRootDir, final String path) throws MaliciousPathException, InvalidBagitFileFormatException { String fixedPath = path; if(path.charAt(0) == '*'){ logger.warn(messages.getString("removing_asterisk")); @@ -37,7 +52,7 @@ static Path createFileFromManifest(final Path bagRootDir, final String path) thr throw new InvalidBagitFileFormatException(MessageFormatter.format(formattedMessage, path).getMessage()); } - if(path.contains("~/")){ + if(path.contains("~/") || path.startsWith("~")){ final String formattedMessage = messages.getString("malicious_path_error"); throw new MaliciousPathException(MessageFormatter.format(formattedMessage, path).getMessage()); } diff --git a/src/main/java/gov/loc/repository/bagit/util/PathUtils.java b/src/main/java/nl/knaw/dans/bagit/util/PathUtils.java similarity index 86% rename from src/main/java/gov/loc/repository/bagit/util/PathUtils.java rename to src/main/java/nl/knaw/dans/bagit/util/PathUtils.java index 744a695d6..38a53d611 100644 --- a/src/main/java/gov/loc/repository/bagit/util/PathUtils.java +++ b/src/main/java/nl/knaw/dans/bagit/util/PathUtils.java @@ -1,13 +1,28 @@ -package gov.loc.repository.bagit.util; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.util; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.DosFileAttributes; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.verify.FileCountAndTotalSizeVistor; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.verify.FileCountAndTotalSizeVistor; /** * Convenience class for dealing with various path issues diff --git a/src/main/java/gov/loc/repository/bagit/verify/AbstractPayloadFileExistsInManifestsVistor.java b/src/main/java/nl/knaw/dans/bagit/verify/AbstractPayloadFileExistsInManifestsVistor.java similarity index 64% rename from src/main/java/gov/loc/repository/bagit/verify/AbstractPayloadFileExistsInManifestsVistor.java rename to src/main/java/nl/knaw/dans/bagit/verify/AbstractPayloadFileExistsInManifestsVistor.java index 2073786e2..39c0fa580 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/AbstractPayloadFileExistsInManifestsVistor.java +++ b/src/main/java/nl/knaw/dans/bagit/verify/AbstractPayloadFileExistsInManifestsVistor.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -11,7 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.util.PathUtils; /** * Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests. diff --git a/src/main/java/nl/knaw/dans/bagit/verify/BagVerifier.java b/src/main/java/nl/knaw/dans/bagit/verify/BagVerifier.java new file mode 100644 index 000000000..3cafaa4f6 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/verify/BagVerifier.java @@ -0,0 +1,330 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.ResourceBundle; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.exceptions.CorruptChecksumException; +import nl.knaw.dans.bagit.exceptions.FileNotInManifestException; +import nl.knaw.dans.bagit.exceptions.FileNotInPayloadDirectoryException; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.InvalidPayloadOxumException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.exceptions.MissingBagitFileException; +import nl.knaw.dans.bagit.exceptions.MissingPayloadDirectoryException; +import nl.knaw.dans.bagit.exceptions.MissingPayloadManifestException; +import nl.knaw.dans.bagit.exceptions.PayloadOxumDoesNotExistException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; +import nl.knaw.dans.bagit.exceptions.VerificationException; +import nl.knaw.dans.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.hash.Hasher.HashOptions; +import nl.knaw.dans.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.hash.SupportedAlgorithm; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Responsible for verifying if a bag is valid, complete + */ +public final class BagVerifier implements AutoCloseable{ + private static final Logger logger = LoggerFactory.getLogger(BagVerifier.class); + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); + + private final ManifestVerifier manifestVerifier; + private final ExecutorService executor; + private Integer chunkSize; + private Integer maxRetries; + private Integer retrySleepMs; + private Integer maxRedirects; + private Boolean fallBackToFullStreamOnRangeFail; + + /** + * Create a BagVerifier with a cached thread pool and a + * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} + */ + public BagVerifier(){ + this(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()), new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); + } + + /** + * Create a BagVerifier with a cached thread pool and a custom mapping + * + * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm + */ + public BagVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){ + this(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()), nameMapping); + } + + /** + * Create a BagVerifier with a custom thread pool and a + * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} + * + * @param executor the thread pool to use when doing work + */ + public BagVerifier(final ExecutorService executor){ + this(executor, new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); + } + + /** + * Create a BagVerifier with a custom thread pool and a custom mapping + * + * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm + * @param executor the thread pool to use when doing work + */ + public BagVerifier(final ExecutorService executor, final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){ + manifestVerifier = new ManifestVerifier(nameMapping, executor); + this.executor = executor; + } + + @Override + public void close() throws SecurityException{ + //shutdown the thread pool so the resource isn't leaked + executor.shutdown(); + manifestVerifier.close(); + } + + public void setChunkSize(final int chunkSize) { + this.chunkSize = chunkSize; + } + + public void setMaxRetries(final int maxRetries) { + this.maxRetries = maxRetries; + } + + public void setRetrySleepMs(final int retrySleepMs) { + this.retrySleepMs = retrySleepMs; + } + + public void setMaxRedirects(final int maxRedirects) { + this.maxRedirects = maxRedirects; + } + + public void setFallBackToFullStreamOnRangeFail(final boolean fallBackToFullStreamOnRangeFail) { + this.fallBackToFullStreamOnRangeFail = fallBackToFullStreamOnRangeFail; + } + + /** + * Determine if we can quickly verify by comparing the number of files and the total number of bytes expected + * + * @param bag the {@link Bag} object you wish to check + * @return true if the bag can be quickly verified + */ + public static boolean canQuickVerify(final Bag bag){ + return QuickVerifier.canQuickVerify(bag); + } + + /** + * Quickly verify by comparing the number of files and the total number of bytes expected + * + * @param bag the bag to verify by payload-oxum + * + * @throws IOException if there is an error reading a file + * @throws InvalidPayloadOxumException if either the total bytes or the number of files + * calculated for the payload directory of the bag is different than the supplied values + * @throws PayloadOxumDoesNotExistException if the bag does not contain a payload-oxum. + * To check, run {@link BagVerifier#canQuickVerify} + */ + public static void quicklyVerify(final Bag bag) throws IOException, InvalidPayloadOxumException{ + QuickVerifier.quicklyVerify(bag); + } + + /** + * See https://tools.ietf.org/html/draft-kunze-bagit-13#section-3
+ * A bag is valid if the bag is complete and every checksum has been + * verified against the contents of its corresponding file. + * + * @param bag the {@link Bag} object to check + * @param ignoreHiddenFiles ignore hidden files unless explicitly listed in manifest(s) + * + * @throws CorruptChecksumException when the computed hash doesn't match given hash + * @throws IOException if there was an error with the file + * @throws FileNotInManifestException if a file is found in the payload directory but not in manifest(s) + * @throws MissingPayloadManifestException if there is not at least one payload manifest + * @throws MissingBagitFileException if there is no bagit.txt file + * @throws MissingPayloadDirectoryException if there is no /data directory + * @throws FileNotInPayloadDirectoryException if a manifest lists a file but it is not in the payload directory + * @throws InterruptedException if the threads are interrupted when checking if all files are listed in manifest(s) + * @throws MaliciousPathException if there is path that is referenced in the manifest that is outside the bag root directory + * @throws VerificationException some other exception happened during processing so capture it here. + * @throws UnsupportedAlgorithmException if the manifest uses a algorithm that isn't supported + * @throws InvalidBagitFileFormatException if the manifest is not formatted properly + */ + public void isValid(final Bag bag, final boolean ignoreHiddenFiles) throws IOException, FileNotInManifestException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + isValid(bag, ignoreHiddenFiles, false); + } + + public void isValid(final Bag bag, final boolean ignoreHiddenFiles, final boolean allowHoley) throws IOException, FileNotInManifestException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + isValid(bag, ignoreHiddenFiles, allowHoley, null); + } + + public void isValid(final Bag bag, final boolean ignoreHiddenFiles, final boolean allowHoley, final Map extraHeaders) throws IOException, FileNotInManifestException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + isValid(bag, ignoreHiddenFiles, allowHoley, extraHeaders, null); + } + + public void isValid(final Bag bag, final boolean ignoreHiddenFiles, final boolean allowHoley, final Map extraHeaders, final Map> urlConfigs) throws IOException, FileNotInManifestException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + isValid(bag, ignoreHiddenFiles, allowHoley, extraHeaders, urlConfigs, null); + } + + public void isValid(final Bag bag, final boolean ignoreHiddenFiles, final boolean allowHoley, final Map extraHeaders, final Map> urlConfigs, final Map> ignoredFetchItems) throws IOException, FileNotInManifestException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + logger.info(messages.getString("checking_bag_is_valid"), bag.getRootDir()); + final boolean holey = allowHoley || !bag.getItemsToFetch().isEmpty(); + isComplete(bag, ignoreHiddenFiles, holey); + + final Map fetchItems = new HashMap<>(); + if (holey) { + for (final FetchItem item : bag.getItemsToFetch()) { + fetchItems.put(item.path, item); + } + } + + logger.debug(messages.getString("checking_payload_checksums")); + for(final Manifest payloadManifest : bag.getPayLoadManifests()){ + final Collection ignoredItemsForAlgorithm = ignoredFetchItems != null ? ignoredFetchItems.get(payloadManifest.getAlgorithm()) : null; + checkHashes(payloadManifest, fetchItems, holey, extraHeaders, urlConfigs, ignoredItemsForAlgorithm); + } + + logger.debug(messages.getString("checking_tag_file_checksums")); + for(final Manifest tagManifest : bag.getTagManifests()){ + checkHashes(tagManifest, null, false, extraHeaders, urlConfigs, null); + } + } + + /* + * Check the supplied checksum hashes against the generated checksum hashes + */ + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + void checkHashes(final Manifest manifest) throws CorruptChecksumException, InterruptedException, VerificationException{ + checkHashes(manifest, null, false); + } + + void checkHashes(final Manifest manifest, final Map fetchItems, final boolean holey) throws CorruptChecksumException, InterruptedException, VerificationException{ + checkHashes(manifest, fetchItems, holey, null); + } + + void checkHashes(final Manifest manifest, final Map fetchItems, final boolean holey, final Map extraHeaders) throws CorruptChecksumException, InterruptedException, VerificationException{ + checkHashes(manifest, fetchItems, holey, extraHeaders, null); + } + + void checkHashes(final Manifest manifest, final Map fetchItems, final boolean holey, final Map extraHeaders, final Map> urlConfigs) throws CorruptChecksumException, InterruptedException, VerificationException{ + checkHashes(manifest, fetchItems, holey, extraHeaders, urlConfigs, null); + } + + void checkHashes(final Manifest manifest, final Map fetchItems, final boolean holey, final Map extraHeaders, final Map> urlConfigs, final Collection ignoredFetchItems) throws CorruptChecksumException, InterruptedException, VerificationException{ + final CountDownLatch latch = new CountDownLatch( manifest.getFileToChecksumMap().size()); + + //TODO maybe return all of these at some point... + final Collection exceptions = Collections.synchronizedCollection(new ArrayList<>()); + final HashOptions hashOptions = getHashOptions(); + + final Collection optimizedIgnoredFetchItems = ignoredFetchItems == null ? null : + (ignoredFetchItems instanceof Set ? ignoredFetchItems : new HashSet<>(ignoredFetchItems)); + + for(final Entry entry : manifest.getFileToChecksumMap().entrySet()){ + executor.execute(new CheckManifestHashesTask(entry, manifest.getAlgorithm().getMessageDigestName(), latch, exceptions, fetchItems, holey, extraHeaders, urlConfigs, hashOptions, optimizedIgnoredFetchItems)); + } + + latch.await(); + + if(!exceptions.isEmpty()){ + final Exception e = exceptions.iterator().next(); + if(e instanceof CorruptChecksumException){ + logger.debug(messages.getString("checksums_not_matching_error"), exceptions.size()); + throw (CorruptChecksumException)e; + } + + throw new VerificationException(e); + } + } + + HashOptions getHashOptions() { + return HashOptions.systemProperties().withOverrides(chunkSize, maxRetries, retrySleepMs, maxRedirects, fallBackToFullStreamOnRangeFail); + } + + /** + * See https://tools.ietf.org/html/draft-kunze-bagit-13#section-3
+ * A bag is complete if
+ *
    + *
  • every element is present + *
  • every file in the payload manifest(s) are present + *
  • every file in the tag manifest(s) are present. Tag files not listed in a tag manifest may be present. + *
  • every file in the data directory must be listed in at least one payload manifest + *
  • each element must comply with the bagit spec + *
+ * + * @param bag the {@link Bag} object to check + * @param ignoreHiddenFiles ignore hidden files unless explicitly listed in manifest(s) + * + * @throws IOException if there was an error with the file + * @throws MissingPayloadManifestException if there is not at least one payload manifest + * @throws MissingBagitFileException if there is no bagit.txt file + * @throws MissingPayloadDirectoryException if there is no /data directory + * @throws FileNotInPayloadDirectoryException if a manifest lists a file but it is not in the payload directory + * @throws InterruptedException if the threads are interrupted when checking if all files are listed in manifest(s) + * @throws MaliciousPathException if there is path that is referenced in the manifest that is outside the bag root directory + * @throws UnsupportedAlgorithmException if the manifest uses a algorithm that isn't supported + * @throws InvalidBagitFileFormatException if the manifest is not formatted properly + */ + public void isComplete(final Bag bag, final boolean ignoreHiddenFiles) throws + IOException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, + FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + isComplete(bag, ignoreHiddenFiles, false); + } + + public void isComplete(final Bag bag, final boolean ignoreHiddenFiles, final boolean allowHoley) throws + IOException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, + FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + logger.info(messages.getString("checking_bag_is_complete"), bag.getRootDir()); + + final boolean holey = allowHoley && !bag.getItemsToFetch().isEmpty(); + + if (!holey) { + MandatoryVerifier.checkFetchItemsExist(bag.getItemsToFetch(), bag.getRootDir()); + } + + MandatoryVerifier.checkBagitFileExists(bag.getRootDir(), bag.getVersion()); + + MandatoryVerifier.checkPayloadDirectoryExists(bag); + + MandatoryVerifier.checkIfAtLeastOnePayloadManifestsExist(bag.getRootDir(), bag.getVersion()); + + manifestVerifier.verifyManifests(bag, ignoreHiddenFiles, holey); + } + + public ExecutorService getExecutor() { + return executor; + } + + public ManifestVerifier getManifestVerifier() { + return manifestVerifier; + } +} diff --git a/src/main/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTask.java b/src/main/java/nl/knaw/dans/bagit/verify/CheckIfFileExistsTask.java similarity index 77% rename from src/main/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTask.java rename to src/main/java/nl/knaw/dans/bagit/verify/CheckIfFileExistsTask.java index 4ace056e3..c4d32e4e2 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTask.java +++ b/src/main/java/nl/knaw/dans/bagit/verify/CheckIfFileExistsTask.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -31,11 +46,10 @@ public CheckIfFileExistsTask(final Path file, final Set missingFiles, fina @Override public void run() { - final boolean existsNormalized = existsNormalized(); final boolean fileExists = Files.exists(file); if(!fileExists){ - if(existsNormalized){ + if(existsNormalized()){ logger.warn(messages.getString("different_normalization_on_filesystem_warning"), file); } else{ diff --git a/src/main/java/nl/knaw/dans/bagit/verify/CheckManifestHashesTask.java b/src/main/java/nl/knaw/dans/bagit/verify/CheckManifestHashesTask.java new file mode 100644 index 000000000..ab657a36b --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/verify/CheckManifestHashesTask.java @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.ResourceBundle; +import java.util.concurrent.CountDownLatch; + +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.exceptions.CorruptChecksumException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import nl.knaw.dans.bagit.hash.Hasher; +import nl.knaw.dans.bagit.hash.Hasher.HashOptions; + +/** + * Checks a give file to make sure the given checksum hash matches the computed checksum hash. + * This is thread safe so you can call many at a time. + */ +@SuppressWarnings("PMD.DoNotUseThreads") +public class CheckManifestHashesTask implements Runnable { + private static final Logger logger = LoggerFactory.getLogger(CheckManifestHashesTask.class); + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); + + private transient final Entry entry; + private transient final CountDownLatch latch; + private transient final Collection exceptions; + private transient final String algorithm; + private transient final Map fetchItems; + private transient final boolean holey; + private transient final Map extraHeaders; + private transient final Map> urlConfigs; + private transient final HashOptions hashOptions; + private transient final Collection ignoredFetchItems; + + public CheckManifestHashesTask(final Entry entry, final String algorithm, final CountDownLatch latch, final Collection exceptions) { + this(entry, algorithm, latch, exceptions, null, false, null, null, null, null); + } + + public CheckManifestHashesTask(final Entry entry, final String algorithm, final CountDownLatch latch, final Collection exceptions, final Map fetchItems, final boolean holey) { + this(entry, algorithm, latch, exceptions, fetchItems, holey, null, null, null, null); + } + + public CheckManifestHashesTask(final Entry entry, final String algorithm, final CountDownLatch latch, final Collection exceptions, final Map fetchItems, final boolean holey, final Map extraHeaders) { + this(entry, algorithm, latch, exceptions, fetchItems, holey, extraHeaders, null, null, null); + } + + public CheckManifestHashesTask(final Entry entry, final String algorithm, final CountDownLatch latch, final Collection exceptions, final Map fetchItems, final boolean holey, final Map extraHeaders, final Map> urlConfigs) { + this(entry, algorithm, latch, exceptions, fetchItems, holey, extraHeaders, urlConfigs, null, null); + } + + public CheckManifestHashesTask(final Entry entry, final String algorithm, final CountDownLatch latch, final Collection exceptions, final Map fetchItems, final boolean holey, final Map extraHeaders, final Map> urlConfigs, final HashOptions hashOptions) { + this(entry, algorithm, latch, exceptions, fetchItems, holey, extraHeaders, urlConfigs, hashOptions, null); + } + + public CheckManifestHashesTask(final Entry entry, final String algorithm, final CountDownLatch latch, final Collection exceptions, final Map fetchItems, final boolean holey, final Map extraHeaders, final Map> urlConfigs, final HashOptions hashOptions, final Collection ignoredFetchItems) { + this.entry = entry; + this.algorithm = algorithm; + this.latch = latch; + this.exceptions = exceptions; + this.fetchItems = fetchItems; + this.holey = holey; + this.extraHeaders = extraHeaders; + this.urlConfigs = urlConfigs; + this.hashOptions = hashOptions; + this.ignoredFetchItems = ignoredFetchItems; + } + + @Override + public void run() { + try { + final MessageDigest messageDigest = MessageDigest.getInstance(algorithm); + checkManifestEntry(entry, messageDigest, algorithm, fetchItems, holey, extraHeaders, urlConfigs, hashOptions, ignoredFetchItems); + } catch (IOException | CorruptChecksumException | NoSuchAlgorithmException e) { + exceptions.add(e); + } + latch.countDown(); + } + + protected static void checkManifestEntry(final Entry entry, final MessageDigest messageDigest, final String algorithm) throws IOException, CorruptChecksumException { + checkManifestEntry(entry, messageDigest, algorithm, null, false, null, null, null, null); + } + + protected static void checkManifestEntry(final Entry entry, final MessageDigest messageDigest, final String algorithm, final Map fetchItems, final boolean allowHoley) throws IOException, CorruptChecksumException { + checkManifestEntry(entry, messageDigest, algorithm, fetchItems, allowHoley, null, null, null, null); + } + + protected static void checkManifestEntry(final Entry entry, final MessageDigest messageDigest, final String algorithm, final Map fetchItems, final boolean allowHoley, final Map extraHeaders) throws IOException, CorruptChecksumException { + checkManifestEntry(entry, messageDigest, algorithm, fetchItems, allowHoley, extraHeaders, null, null, null); + } + + protected static void checkManifestEntry(final Entry entry, final MessageDigest messageDigest, final String algorithm, final Map fetchItems, final boolean allowHoley, final Map extraHeaders, final Map> urlConfigs) throws IOException, CorruptChecksumException { + checkManifestEntry(entry, messageDigest, algorithm, fetchItems, allowHoley, extraHeaders, urlConfigs, null, null); + } + + protected static void checkManifestEntry(final Entry entry, final MessageDigest messageDigest, final String algorithm, final Map fetchItems, final boolean allowHoley, final Map extraHeaders, final Map> urlConfigs, final HashOptions hashOptions) throws IOException, CorruptChecksumException { + checkManifestEntry(entry, messageDigest, algorithm, fetchItems, allowHoley, extraHeaders, urlConfigs, hashOptions, null); + } + + protected static void checkManifestEntry(final Entry entry, final MessageDigest messageDigest, final String algorithm, final Map fetchItems, final boolean allowHoley, final Map extraHeaders, final Map> urlConfigs, final HashOptions hashOptions, final Collection ignoredFetchItems) throws IOException, CorruptChecksumException { + if (Files.exists(entry.getKey())) { + logger.debug(messages.getString("checking_checksums"), entry.getKey(), entry.getValue()); + final String hash = Hasher.hash(entry.getKey(), messageDigest); + logger.debug("computed hash [{}] for file [{}]", hash, entry.getKey()); + if (!hash.equals(entry.getValue())) { + throw new CorruptChecksumException(messages.getString("corrupt_checksum_error"), entry.getKey(), algorithm, entry.getValue(), hash); + } + } else if (allowHoley && fetchItems != null && fetchItems.containsKey(entry.getKey())) { + final FetchItem item = fetchItems.get(entry.getKey()); + if (ignoredFetchItems != null && ignoredFetchItems.contains(item)) { + logger.debug("skipping hashing for file [{}] because it is in the ignored fetch items list", entry.getKey()); + return; + } + logger.debug("File {} does not exist, but it is in fetch.txt, and allowHoley is true. Hashing from URL: {}", entry.getKey(), item.url); + final Map mergedHeaders = mergeHeaders(item.url, extraHeaders, urlConfigs); + final String hash = Hasher.hash(item, messageDigest, mergedHeaders, hashOptions); + logger.debug("computed hash [{}] for url [{}]", hash, item.url); + if (!hash.equals(entry.getValue())) { + throw new CorruptChecksumException(messages.getString("corrupt_checksum_error"), entry.getKey(), algorithm, entry.getValue(), hash); + } + } + //if the file doesn't exist it will be caught by checkAllFilesListedInManifestExist method + } + + private static Map mergeHeaders(final URL url, final Map extraHeaders, final Map> urlConfigs) { + if (urlConfigs == null || urlConfigs.isEmpty()) { + return extraHeaders; + } + + final Map mergedHeaders = new HashMap<>(); + if (extraHeaders != null) { + mergedHeaders.putAll(extraHeaders); + } + + final String urlString = url.toString(); + for (final Entry> configEntry : urlConfigs.entrySet()) { + if (urlString.startsWith(configEntry.getKey())) { + mergedHeaders.putAll(configEntry.getValue()); + } + } + + return mergedHeaders; + } +} diff --git a/src/main/java/gov/loc/repository/bagit/verify/FileCountAndTotalSizeVistor.java b/src/main/java/nl/knaw/dans/bagit/verify/FileCountAndTotalSizeVistor.java similarity index 69% rename from src/main/java/gov/loc/repository/bagit/verify/FileCountAndTotalSizeVistor.java rename to src/main/java/nl/knaw/dans/bagit/verify/FileCountAndTotalSizeVistor.java index 25babc547..ca6d3b212 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/FileCountAndTotalSizeVistor.java +++ b/src/main/java/nl/knaw/dans/bagit/verify/FileCountAndTotalSizeVistor.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.IOException; import java.nio.file.FileVisitResult; diff --git a/src/main/java/gov/loc/repository/bagit/verify/MandatoryVerifier.java b/src/main/java/nl/knaw/dans/bagit/verify/MandatoryVerifier.java similarity index 78% rename from src/main/java/gov/loc/repository/bagit/verify/MandatoryVerifier.java rename to src/main/java/nl/knaw/dans/bagit/verify/MandatoryVerifier.java index e73ed3b02..e7ece22f5 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/MandatoryVerifier.java +++ b/src/main/java/nl/knaw/dans/bagit/verify/MandatoryVerifier.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.File; import java.io.IOException; @@ -8,18 +23,18 @@ import java.util.List; import java.util.ResourceBundle; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.exceptions.FileNotInPayloadDirectoryException; +import nl.knaw.dans.bagit.exceptions.MissingBagitFileException; +import nl.knaw.dans.bagit.exceptions.MissingPayloadDirectoryException; +import nl.knaw.dans.bagit.exceptions.MissingPayloadManifestException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.FetchItem; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.MissingBagitFileException; -import gov.loc.repository.bagit.exceptions.MissingPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.MissingPayloadManifestException; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.util.PathUtils; /** * Responsible for checking all things related to mandatory files for the bagit specification @@ -84,7 +99,7 @@ public static void checkPayloadDirectoryExists(final Bag bag) throws MissingPayl final Path dataDir = PathUtils.getDataDir(bag); if(!Files.exists(dataDir)){ - throw new MissingPayloadDirectoryException(messages.getString("file_should_exist_error")); + throw new MissingPayloadDirectoryException(messages.getString("file_should_exist_error"), dataDir); } } diff --git a/src/main/java/nl/knaw/dans/bagit/verify/ManifestVerifier.java b/src/main/java/nl/knaw/dans/bagit/verify/ManifestVerifier.java new file mode 100644 index 000000000..1eb5d10a8 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/verify/ManifestVerifier.java @@ -0,0 +1,260 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.ResourceBundle; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.exceptions.FileNotInPayloadDirectoryException; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.helpers.MessageFormatter; + +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.util.PathUtils; + +/** + * Responsible for all things related to the manifest during verification. + */ +public class ManifestVerifier implements AutoCloseable{ + private static final Logger logger = LoggerFactory.getLogger(ManifestVerifier.class); + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); + + private transient final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping; + private transient final ExecutorService executor; + + /** + * Create a PayloadVerifier using a cached thread pool and the + * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} mapping + */ + public ManifestVerifier(){ + this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); + } + + /** + * Create a PayloadVerifier using a cached thread pool and a custom mapping + * + * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm + */ + public ManifestVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping) { + this(nameMapping, Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); + } + + /** + * Create a PayloadVerifier using a custom thread pool and the + * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} mapping + * + * @param executor the thread pool to use when doing work + */ + public ManifestVerifier(final ExecutorService executor) { + this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), executor); + } + + /** + * Create a PayloadVerifier using a custom thread pool and a custom mapping + * + * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm + * @param executor the thread pool to use when doing work + */ + public ManifestVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping, final ExecutorService executor) { + this.nameMapping = nameMapping; + this.executor = executor; + } + + @Override + public void close() throws SecurityException{ + //shutdown the thread pool so the resource isn't leaked + executor.shutdown(); + } + + /** + * Verify that all the files in the payload directory are listed in the payload manifest and + * all files listed in all manifests exist. + * + * @param bag the bag to check to check + * @param ignoreHiddenFiles to ignore hidden files unless they are specifically listed in a manifest + * + * @throws IOException if there is a problem reading a file + * @throws MaliciousPathException the path in the manifest was specifically crafted to cause harm + * @throws UnsupportedAlgorithmException if the algorithm used for the manifest is unsupported + * @throws InvalidBagitFileFormatException if any of the manifests don't conform to the bagit specification + * @throws FileNotInPayloadDirectoryException if a file is listed in a manifest but doesn't exist in the payload directory + * @throws InterruptedException if a thread is interrupted while doing work + */ + /* + * Verify that all the files in the payload directory are listed in the payload manifest and + * all files listed in all manifests exist. + * + * @param bag the bag to check to check + * @param ignoreHiddenFiles to ignore hidden files unless they are specifically listed in a manifest + * + * @throws IOException if there is a problem reading a file + * @throws MaliciousPathException the path in the manifest was specifically crafted to cause harm + * @throws UnsupportedAlgorithmException if the algorithm used for the manifest is unsupported + * @throws InvalidBagitFileFormatException if any of the manifests don't conform to the bagit specification + * @throws FileNotInPayloadDirectoryException if a file is listed in a manifest but doesn't exist in the payload directory + * @throws InterruptedException if a thread is interrupted while doing work + */ + public void verifyManifests(final Bag bag, final boolean ignoreHiddenFiles) + throws IOException, MaliciousPathException, UnsupportedAlgorithmException, + InvalidBagitFileFormatException, FileNotInPayloadDirectoryException, InterruptedException { + verifyManifests(bag, ignoreHiddenFiles, false); + } + + public void verifyManifests(final Bag bag, final boolean ignoreHiddenFiles, final boolean holey) + throws IOException, MaliciousPathException, UnsupportedAlgorithmException, + InvalidBagitFileFormatException, FileNotInPayloadDirectoryException, InterruptedException { + + checkAlgorithms(bag.getPayLoadManifests()); + checkAlgorithms(bag.getTagManifests()); + + final Set payloadFiles = getFilesListedInPayloadManifests(bag); + final Set tagFiles = getFilesListedInTagManifests(bag); + + checkAllFilesListedInManifestExist(payloadFiles, holey, bag); + checkAllFilesListedInManifestExist(tagFiles, false, bag); + + final Set allFilesListedInManifests = new HashSet<>(payloadFiles); + allFilesListedInManifests.addAll(tagFiles); + + if (bag.getVersion().isOlder(new Version(1, 0))) { + checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(allFilesListedInManifests, PathUtils.getDataDir(bag), ignoreHiddenFiles); + } else { + checkAllFilesInPayloadDirAreListedInAllManifests(bag.getPayLoadManifests(), PathUtils.getDataDir(bag), ignoreHiddenFiles); + } + } + + private void checkAlgorithms(final Set manifests) throws UnsupportedAlgorithmException { + for (final Manifest manifest : manifests) { + if (nameMapping.getSupportedAlgorithm(manifest.getAlgorithm().getBagitName()) == null) { + throw new UnsupportedAlgorithmException(messages.getString("unsupported_algorithm_error"), manifest.getAlgorithm().getBagitName(), null); + } + } + } + + /* + * get all the files listed in the payload manifests + */ + private Set getFilesListedInPayloadManifests(final Bag bag) + throws IOException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException { + logger.debug(messages.getString("all_files_in_manifests")); + final Set filesListedInManifests = new HashSet<>(); + + for (final Manifest manifest : bag.getPayLoadManifests()) { + filesListedInManifests.addAll(manifest.getFileToChecksumMap().keySet()); + } + + return filesListedInManifests; + } + + /* + * get all the files listed in the tag manifests + */ + private Set getFilesListedInTagManifests(final Bag bag) + throws IOException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException { + logger.debug(messages.getString("all_files_in_manifests")); + final Set filesListedInManifests = new HashSet<>(); + + for (final Manifest manifest : bag.getTagManifests()) { + filesListedInManifests.addAll(manifest.getFileToChecksumMap().keySet()); + } + + return filesListedInManifests; + } + + /* + * Make sure all the listed files actually exist + */ + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + private void checkAllFilesListedInManifestExist(final Set files, final boolean holey, final Bag bag) throws FileNotInPayloadDirectoryException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(files.size()); + final Set missingFiles = new ConcurrentSkipListSet<>(); + + final Map fetchUrls = new HashMap<>(); + if (holey) { + for (final FetchItem item : bag.getItemsToFetch()) { + fetchUrls.put(item.path, item.url); + } + } + + logger.info(messages.getString("check_all_files_in_manifests_exist")); + for (final Path file : files) { + if (holey && fetchUrls.containsKey(file)) { + // Not actually checking that the file can be downloaded. That will be done when calculating the checksums later on. + latch.countDown(); + } else { + executor.execute(new CheckIfFileExistsTask(file, missingFiles, latch)); + } + } + + latch.await(); + + if (!missingFiles.isEmpty()) { + final String formattedMessage = messages.getString("missing_payload_files_error"); + throw new FileNotInPayloadDirectoryException(MessageFormatter.format(formattedMessage, missingFiles).getMessage()); + } + } + + /* + * Make sure all the listed files actually exist + */ + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + private void checkAllFilesListedInManifestExist(final Set files) throws FileNotInPayloadDirectoryException, InterruptedException { + checkAllFilesListedInManifestExist(files, false, null); + } + + /* + * Make sure all files in the directory are in at least 1 manifest + */ + private static void checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(final Set filesListedInManifests, + final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException { + logger.debug(messages.getString("checking_file_in_at_least_one_manifest"), payloadDir); + if (Files.exists(payloadDir)) { + Files.walkFileTree(payloadDir, + new PayloadFileExistsInAtLeastOneManifestVistor(filesListedInManifests, ignoreHiddenFiles)); + } + } + + /* + * as per the bagit-spec 1.0+ all files have to be listed in all manifests + */ + private static void checkAllFilesInPayloadDirAreListedInAllManifests(final Set payLoadManifests, + final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException { + logger.debug(messages.getString("checking_file_in_all_manifests"), payloadDir); + if (Files.exists(payloadDir)) { + Files.walkFileTree(payloadDir, new PayloadFileExistsInAllManifestsVistor(payLoadManifests, ignoreHiddenFiles)); + } + } +} diff --git a/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAllManifestsVistor.java b/src/main/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAllManifestsVistor.java similarity index 60% rename from src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAllManifestsVistor.java rename to src/main/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAllManifestsVistor.java index 3763f913c..82d5cbb66 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAllManifestsVistor.java +++ b/src/main/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAllManifestsVistor.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.nio.file.FileVisitResult; import java.nio.file.Files; @@ -7,11 +22,10 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.Set; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.exceptions.FileNotInManifestException; import org.slf4j.helpers.MessageFormatter; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.exceptions.FileNotInManifestException; - /** * Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests. */ @@ -24,7 +38,7 @@ public PayloadFileExistsInAllManifestsVistor(final Set manifests, fina } @Override - public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs)throws FileNotInManifestException{ + public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs)throws FileNotInManifestException { if(Files.isRegularFile(path)){ for(final Manifest manifest : manifests){ if(!manifest.getFileToChecksumMap().keySet().contains(path.normalize())){ diff --git a/src/main/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java b/src/main/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java new file mode 100644 index 000000000..f3835ea0c --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ResourceBundle; +import java.util.Set; + +import nl.knaw.dans.bagit.exceptions.FileNotInManifestException; +import org.slf4j.helpers.MessageFormatter; + +/** + * Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests. + */ +public class PayloadFileExistsInAtLeastOneManifestVistor extends AbstractPayloadFileExistsInManifestsVistor { + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); + private transient final Set filesListedInManifests; + + public PayloadFileExistsInAtLeastOneManifestVistor(final Set filesListedInManifests, final boolean ignoreHiddenFiles) { + super(ignoreHiddenFiles); + this.filesListedInManifests = filesListedInManifests; + } + + @Override + public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs)throws IOException, FileNotInManifestException { + if(Files.isHidden(path) && ignoreHiddenFiles){ + logger.debug(messages.getString("skipping_hidden_file"), path); + } + else { + if(Files.isRegularFile(path) && !filesListedInManifests.contains(path.normalize())){ + final String formattedMessage = messages.getString("file_not_in_any_manifest_error"); + throw new FileNotInManifestException(MessageFormatter.format(formattedMessage, path).getMessage()); + } + logger.debug(messages.getString("file_in_at_least_one_manifest"), path); + } + return FileVisitResult.CONTINUE; + } + +} diff --git a/src/main/java/gov/loc/repository/bagit/verify/QuickVerifier.java b/src/main/java/nl/knaw/dans/bagit/verify/QuickVerifier.java similarity index 80% rename from src/main/java/gov/loc/repository/bagit/verify/QuickVerifier.java rename to src/main/java/nl/knaw/dans/bagit/verify/QuickVerifier.java index 2b6689eb1..993521089 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/QuickVerifier.java +++ b/src/main/java/nl/knaw/dans/bagit/verify/QuickVerifier.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.IOException; import java.nio.file.Files; @@ -6,14 +21,14 @@ import java.util.ResourceBundle; import java.util.AbstractMap.SimpleImmutableEntry; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.exceptions.InvalidPayloadOxumException; +import nl.knaw.dans.bagit.exceptions.PayloadOxumDoesNotExistException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.exceptions.InvalidPayloadOxumException; -import gov.loc.repository.bagit.exceptions.PayloadOxumDoesNotExistException; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.util.PathUtils; /** * responsible for all things related to quick verification. Quick verification does not @@ -59,7 +74,7 @@ private static String getPayloadOxum(final Bag bag){ * @param bag the bag to verify by payload-oxum * * @throws IOException if there is an error reading a file - * @throws InvalidPayloadOxumException if either the total bytes or the number of files + * @throws InvalidPayloadOxumException if either the total bytes or the number of files * calculated for the payload directory of the bag is different than the supplied values * @throws PayloadOxumDoesNotExistException if the bag does not contain a payload-oxum. * To check, run {@link BagVerifier#canQuickVerify} diff --git a/src/main/java/gov/loc/repository/bagit/writer/BagWriter.java b/src/main/java/nl/knaw/dans/bagit/writer/BagWriter.java similarity index 85% rename from src/main/java/gov/loc/repository/bagit/writer/BagWriter.java rename to src/main/java/nl/knaw/dans/bagit/writer/BagWriter.java index 8c87a4ff4..7f1ba2e4a 100644 --- a/src/main/java/gov/loc/repository/bagit/writer/BagWriter.java +++ b/src/main/java/nl/knaw/dans/bagit/writer/BagWriter.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.io.IOException; import java.nio.file.Files; @@ -10,14 +25,13 @@ import java.util.ResourceBundle; import java.util.Set; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.Hasher; +import nl.knaw.dans.bagit.util.PathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.hash.Hasher; -import gov.loc.repository.bagit.util.PathUtils; - /** * responsible for writing out a {@link Bag} */ diff --git a/src/main/java/nl/knaw/dans/bagit/writer/BagitFileWriter.java b/src/main/java/nl/knaw/dans/bagit/writer/BagitFileWriter.java new file mode 100644 index 000000000..5920bd2c6 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/writer/BagitFileWriter.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ResourceBundle; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import nl.knaw.dans.bagit.domain.Version; +import java.io.BufferedWriter; + +/** + * Responsible for writing the bagit.txt to the filesystem + */ +public final class BagitFileWriter { + private static final Logger logger = LoggerFactory.getLogger(BagitFileWriter.class); + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); + + private BagitFileWriter(){ + //intentionally left empty + } + + /** + * Write the bagit.txt file in required UTF-8 encoding. + * + * @param version the version of the bag to write out + * @param encoding the encoding of the tag files + * @param outputDir the root of the bag + * + * @throws IOException if there was a problem writing the file + */ + public static void writeBagitFile(final Version version, final Charset encoding, final Path outputDir) throws IOException{ + final Path bagitPath = outputDir.resolve("bagit.txt"); + logger.debug(messages.getString("write_bagit_file_to_path"), outputDir); + + try (BufferedWriter writer = Files.newBufferedWriter(bagitPath, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE)) { + + final String firstLine = "BagIt-Version: " + version + System.lineSeparator(); + logger.debug(messages.getString("writing_line_to_file"), firstLine, bagitPath); + writer.append(firstLine); + + final String secondLine = "Tag-File-Character-Encoding: " + encoding + System.lineSeparator(); + logger.debug(messages.getString("writing_line_to_file"), secondLine, bagitPath); + writer.append(secondLine); + } + } +} diff --git a/src/main/java/gov/loc/repository/bagit/writer/FetchWriter.java b/src/main/java/nl/knaw/dans/bagit/writer/FetchWriter.java similarity index 62% rename from src/main/java/gov/loc/repository/bagit/writer/FetchWriter.java rename to src/main/java/nl/knaw/dans/bagit/writer/FetchWriter.java index 30c4c780c..3a1018b68 100644 --- a/src/main/java/gov/loc/repository/bagit/writer/FetchWriter.java +++ b/src/main/java/nl/knaw/dans/bagit/writer/FetchWriter.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.io.IOException; import java.nio.charset.Charset; @@ -8,10 +23,11 @@ import java.util.List; import java.util.ResourceBundle; +import nl.knaw.dans.bagit.domain.FetchItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.FetchItem; +import java.io.BufferedWriter; /** * Responsible for writing out the list of {@link FetchItem} to the fetch.txt file on the filesystem @@ -37,11 +53,14 @@ private FetchWriter(){ public static void writeFetchFile(final List itemsToFetch, final Path outputDir, final Path bagitRootDir, final Charset charsetName) throws IOException{ logger.debug(messages.getString("writing_fetch_file_to_path"), outputDir); final Path fetchFilePath = outputDir.resolve("fetch.txt"); - - for(final FetchItem item : itemsToFetch){ - final String line = formatFetchLine(item, bagitRootDir); - logger.debug(messages.getString("writing_line_to_file"), line, fetchFilePath); - Files.write(fetchFilePath, line.getBytes(charsetName), StandardOpenOption.APPEND, StandardOpenOption.CREATE); + + try (BufferedWriter writer = Files.newBufferedWriter(fetchFilePath, charsetName, + StandardOpenOption.APPEND, StandardOpenOption.CREATE)) { + for (final FetchItem item : itemsToFetch) { + final String line = formatFetchLine(item, bagitRootDir); + logger.debug(messages.getString("writing_line_to_file"), line, fetchFilePath); + writer.append(line); + } } } diff --git a/src/main/java/gov/loc/repository/bagit/writer/ManifestWriter.java b/src/main/java/nl/knaw/dans/bagit/writer/ManifestWriter.java similarity index 64% rename from src/main/java/gov/loc/repository/bagit/writer/ManifestWriter.java rename to src/main/java/nl/knaw/dans/bagit/writer/ManifestWriter.java index 774b44bbc..7b0188d42 100644 --- a/src/main/java/gov/loc/repository/bagit/writer/ManifestWriter.java +++ b/src/main/java/nl/knaw/dans/bagit/writer/ManifestWriter.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.io.IOException; import java.nio.charset.Charset; @@ -9,10 +24,11 @@ import java.util.ResourceBundle; import java.util.Set; +import nl.knaw.dans.bagit.domain.Manifest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.Manifest; +import java.io.BufferedWriter; /** * Responsible for writing out a {@link Manifest} to the filesystem @@ -65,14 +81,16 @@ private static void writeManifests(final Set manifests, final Path out Files.deleteIfExists(manifestPath); Files.createFile(manifestPath); - - for(final Entry entry : manifest.getFileToChecksumMap().entrySet()){ - //there are 2 spaces between the checksum and the path so that the manifests are compatible with the md5sum tools available on most unix systems. - //This may cause problems on windows due to it being text mode, in which case either replace with a * or try verifying in binary mode with --binary - final String line = entry.getValue() + " " + RelativePathWriter.formatRelativePathString(relativeTo, entry.getKey()); - logger.debug(messages.getString("writing_line_to_file"), line, manifestPath); - Files.write(manifestPath, line.getBytes(charsetName), - StandardOpenOption.APPEND, StandardOpenOption.CREATE); + + try (BufferedWriter writer = Files.newBufferedWriter(manifestPath, charsetName, + StandardOpenOption.APPEND, StandardOpenOption.CREATE)) { + for (final Entry entry : manifest.getFileToChecksumMap().entrySet()) { + //there are 2 spaces between the checksum and the path so that the manifests are compatible with the md5sum tools available on most unix systems. + //This may cause problems on windows due to it being text mode, in which case either replace with a * or try verifying in binary mode with --binary + final String line = entry.getValue() + " " + RelativePathWriter.formatRelativePathString(relativeTo, entry.getKey()); + logger.debug(messages.getString("writing_line_to_file"), line, manifestPath); + writer.append(line); + } } } } diff --git a/src/main/java/nl/knaw/dans/bagit/writer/MetadataWriter.java b/src/main/java/nl/knaw/dans/bagit/writer/MetadataWriter.java new file mode 100644 index 000000000..9467e2173 --- /dev/null +++ b/src/main/java/nl/knaw/dans/bagit/writer/MetadataWriter.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ResourceBundle; +import java.util.AbstractMap.SimpleImmutableEntry; + +import nl.knaw.dans.bagit.domain.Metadata; +import nl.knaw.dans.bagit.domain.Version; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Responsible for writing out the bag {@link Metadata} to the filesystem + */ +public final class MetadataWriter { + private static final Logger logger = LoggerFactory.getLogger(MetadataWriter.class); + private static final Version VERSION_0_95 = new Version(0, 95); + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); + + private MetadataWriter(){ + //intentionall left empty + } + + /** + * Write the bag-info.txt (or package-info.txt) file to the specified outputDir with specified encoding (charsetName) + * + * @param metadata the key value pair info in the bag-info.txt file + * @param version the version of the bag you are writing + * @param outputDir the root of the bag + * @param charsetName the name of the encoding for the file + * + * @throws IOException if there was a problem writing a file + */ + public static void writeBagMetadata(final Metadata metadata, final Version version, final Path outputDir, final Charset charsetName) throws IOException{ + Path bagInfoFilePath = outputDir.resolve("bag-info.txt"); + if(version.isSameOrOlder(VERSION_0_95)){ + bagInfoFilePath = outputDir.resolve("package-info.txt"); + } + logger.debug(messages.getString("writing_metadata_to_path"), bagInfoFilePath.getFileName(), outputDir); + + Files.deleteIfExists(bagInfoFilePath); + final StringBuilder lines = new StringBuilder(); + + for(final SimpleImmutableEntry entry : metadata.getAll()){ + final String key = entry.getKey(); + String value = entry.getValue(); + value = value.replaceAll("[^\\x09\\x20-\\x7E\\x0A\\x0D\\x80-\\uFFFF]", ""); + lines.append(formatLine(key, value, version)).append(System.lineSeparator()); + } + + logger.debug(messages.getString("writing_line_to_file"), lines.toString(), bagInfoFilePath); + Files.write(bagInfoFilePath, lines.toString().getBytes(charsetName), + StandardOpenOption.APPEND, StandardOpenOption.CREATE); + } + + private static String formatLine(final String key, final String value, final Version version) { + if (version.isSameOrOlder(VERSION_0_95)) { + final String fullLine = key + ": " + value; + return fullLine.replaceAll("(\\r\\n|\\r|\\n)", "$1 "); + } + + final StringBuilder sb = new StringBuilder(); + final int keyPrefixLength = key.length() + 2; // "Key: " + sb.append(key).append(": "); + final String[] parts = value.split("(\\r\\n|\\r|\\n)", -1); + + for (int i = 0; i < parts.length; i++) { + String line = parts[i]; + if (i > 0) { + sb.append(System.lineSeparator()); + line = " " + line; + } + + final int maxLength = i == 0 ? 79 - keyPrefixLength : 78; // Continuation lines have " " prefix + // Check if it's already "well-wrapped" or short + if (line.length() > maxLength) { + sb.append(wrapLine(line, i == 0, keyPrefixLength)); + } else { + sb.append(line); + } + } + return sb.toString(); + } + + private static String wrapLine(final String line, final boolean isFirstLine, final int keyPrefixLength) { + final StringBuilder sb = new StringBuilder(); + int start = 0; + boolean firstWrap = true; + + while (start < line.length()) { + int maxLength = 78; // Indented line starts with " ", so 78 more chars = 79 + if (firstWrap && isFirstLine) { + maxLength = 79 - keyPrefixLength; + } + + int end = Math.min(start + maxLength, line.length()); + + if (end < line.length()) { + int lastSpace = line.lastIndexOf(' ', end); + if (lastSpace > start) { + end = lastSpace; + } + } + + String sub = line.substring(start, end); + if (start > 0) { + sb.append(System.lineSeparator()); + sb.append(" "); + } + sb.append(sub); + + start = end; + while (start < line.length() && line.charAt(start) == ' ') { + start++; + } + firstWrap = false; + } + + return sb.toString(); + } +} diff --git a/src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java b/src/main/java/nl/knaw/dans/bagit/writer/PayloadWriter.java similarity index 80% rename from src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java rename to src/main/java/nl/knaw/dans/bagit/writer/PayloadWriter.java index 1dee4a4ac..6471a57cc 100644 --- a/src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java +++ b/src/main/java/nl/knaw/dans/bagit/writer/PayloadWriter.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.io.IOException; import java.nio.file.Files; @@ -9,14 +24,13 @@ import java.util.ResourceBundle; import java.util.Set; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.domain.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.FetchItem; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.domain.Version; - /** * Responsible for writing out the bag payload to the filesystem */ diff --git a/src/main/java/gov/loc/repository/bagit/writer/RelativePathWriter.java b/src/main/java/nl/knaw/dans/bagit/writer/RelativePathWriter.java similarity index 51% rename from src/main/java/gov/loc/repository/bagit/writer/RelativePathWriter.java rename to src/main/java/nl/knaw/dans/bagit/writer/RelativePathWriter.java index 437c29303..ffe1150c9 100644 --- a/src/main/java/gov/loc/repository/bagit/writer/RelativePathWriter.java +++ b/src/main/java/nl/knaw/dans/bagit/writer/RelativePathWriter.java @@ -1,8 +1,23 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.nio.file.Path; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.util.PathUtils; /** * Convenience class for writing a relative path diff --git a/src/main/resources/MessageBundle.properties b/src/main/resources/MessageBundle.properties index 4973bd2e8..8a654e5f9 100644 --- a/src/main/resources/MessageBundle.properties +++ b/src/main/resources/MessageBundle.properties @@ -4,10 +4,13 @@ fetch_allowed=Are fetch files allowed? [{}] serialization_allowed=Serialization is: [{}] parsing_bagit_profile_info_section=Parsing the BagIt-Profile-Info section +parsing_mandatory_tags_of_bagit_profile_info_section=Parsing mandatory tags of the BagIt-Profile-Info section +parsing_optional_tags_of_bagit_profile_info_section=Parsing optional tags of the BagIt-Profile-Info section identifier=Identifier is [{}] source_organization=Source-Organization is [{}] contact_name=Contact-Name is [{}] contact_email=Contact-Email is [{}] +contact_phone=Contact-Phone is [{}] external_description=External-Description is [{}] version=Version is [{}] parsing_bag_info=Parsing the Bag-Info section @@ -182,6 +185,9 @@ file_not_in_manifest_error=File [{}] is in the payload directory but isn't liste file_in_all_manifests=[{}] is in all manifests. file_not_in_any_manifest_error=File [{}] is in the payload directory but isn't listed in any manifest! +#for PayloadFileExistsInAtLeastOneManifestVistor.java +file_in_at_least_one_manifest="[{}] is in at least one manifest" + #for PayloadVerifier.java all_files_in_manifests=Getting all files listed in the manifest(s). get_listing_in_manifest=Getting files and checksums listed in [{}]. diff --git a/src/main/resources/MessageBundle_ar.properties b/src/main/resources/MessageBundle_ar.properties index 34868ae7f..81353903e 100644 --- a/src/main/resources/MessageBundle_ar.properties +++ b/src/main/resources/MessageBundle_ar.properties @@ -5,10 +5,13 @@ fetch_allowed=Are fetch files allowed? [{}] serialization_allowed=Serialization is\: [{}] parsing_bagit_profile_info_section=Parsing the BagIt-Profile-Info section +parsing_mandatory_tags_of_bagit_profile_info_section=Parsing mandatory tags of the BagIt-Profile-Info section +parsing_optional_tags_of_bagit_profile_info_section=Parsing optional tags of the BagIt-Profile-Info section identifier=Identifier is [{}] source_organization=Source-Organization is [{}] contact_name=Contact-Name is [{}] contact_email=Contact-Email is [{}] +contact_phone=Contact-Phone is [{}] external_description=External-Description is [{}] version=Version is [{}] parsing_bag_info=Parsing the Bag-Info section diff --git a/src/main/resources/MessageBundle_de_DE.properties b/src/main/resources/MessageBundle_de_DE.properties index e7488a22d..7386be7f6 100644 --- a/src/main/resources/MessageBundle_de_DE.properties +++ b/src/main/resources/MessageBundle_de_DE.properties @@ -4,10 +4,13 @@ fetch_allowed=Sind Fetch Dateien erlaubt? [{}] serialization_allowed=Serialisierung ist: [{}] parsing_bagit_profile_info_section=Lese Abschnitt BagIt-Profile-Info +parsing_mandatory_tags_of_bagit_profile_info_section=Parse die ben\u00f6tigten Tags im Abschnitt BagIt-Profile-Info +parsing_optional_tags_of_bagit_profile_info_section=Parse die optionalen Tags im Abschnitt BagIt-Profile-Info identifier=Identifier hat den Wert [{}] source_organization=Source-Organization hat den Wert [{}] contact_name=Contact-Name hat den Wert [{}] contact_email=Contact-Email hat den Wert [{}] +contact_phone=Contact-Phone hat den Wert [{}] external_description=External-Description hat den Wert [{}] version=Version hat den Wert [{}] parsing_bag_info=Lese Abschnitt Bag-Info diff --git a/src/main/resources/MessageBundle_es_ES.properties b/src/main/resources/MessageBundle_es_ES.properties index c63efd1d3..38af9cee7 100644 --- a/src/main/resources/MessageBundle_es_ES.properties +++ b/src/main/resources/MessageBundle_es_ES.properties @@ -5,10 +5,13 @@ fetch_allowed=\u00bfSe permiten archivos de recuperaci\u00f3n? [{}] serialization_allowed=Serializaci\u00f3n es\: [{}] parsing_bagit_profile_info_section=An\u00e1lisis de la secci\u00f3n de informaci\u00f3n de perfil BagIt +parsing_mandatory_tags_of_bagit_profile_info_section=An\u00e1lisis de las etiquetas obligatorias de la secci\u00f3n de informaci\u00f3n de perfil BagIt +parsing_optional_tags_of_bagit_profile_info_section=An\u00e1lisis de las etiquetas opcionales de la secci\u00f3n de informaci\u00f3n de perfil BagIt identifier=Identificador es [{}] source_organization=Organizaci\u00f3n de la fuente es [{}] contact_name=Nombre del contacto es [{}] contact_email=Email del contacto es [{}] +contact_phone=Tel\u00e9fono de contacto es [{}] external_description=Descripci\u00f3n externa es [{}] version=La versi\u00f3n es [{}] parsing_bag_info=An\u00e1lisis de la secci\u00f3n de la bag-info diff --git a/src/main/resources/MessageBundle_zh.properties b/src/main/resources/MessageBundle_zh.properties index 34868ae7f..81353903e 100644 --- a/src/main/resources/MessageBundle_zh.properties +++ b/src/main/resources/MessageBundle_zh.properties @@ -5,10 +5,13 @@ fetch_allowed=Are fetch files allowed? [{}] serialization_allowed=Serialization is\: [{}] parsing_bagit_profile_info_section=Parsing the BagIt-Profile-Info section +parsing_mandatory_tags_of_bagit_profile_info_section=Parsing mandatory tags of the BagIt-Profile-Info section +parsing_optional_tags_of_bagit_profile_info_section=Parsing optional tags of the BagIt-Profile-Info section identifier=Identifier is [{}] source_organization=Source-Organization is [{}] contact_name=Contact-Name is [{}] contact_email=Contact-Email is [{}] +contact_phone=Contact-Phone is [{}] external_description=External-Description is [{}] version=Version is [{}] parsing_bag_info=Parsing the Bag-Info section diff --git a/src/main/resources/ruleset.xml b/src/main/resources/ruleset.xml new file mode 100644 index 000000000..1cc563184 --- /dev/null +++ b/src/main/resources/ruleset.xml @@ -0,0 +1,35 @@ + + + + + PMD ruleset for dans-bagit-lib + + + + + + + + + + + diff --git a/src/site/site.xml b/src/site/site.xml new file mode 100644 index 000000000..de0a41e31 --- /dev/null +++ b/src/site/site.xml @@ -0,0 +1,30 @@ + + + + + org.apache.maven.skins + maven-fluido-skin + 2.0.0-M9 + + + + + diff --git a/src/test/java/gov/loc/repository/bagit/TempFolderTest.java b/src/test/java/gov/loc/repository/bagit/TempFolderTest.java deleted file mode 100644 index 591345d2a..000000000 --- a/src/test/java/gov/loc/repository/bagit/TempFolderTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package gov.loc.repository.bagit; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -abstract public class TempFolderTest { - protected Path folder; - - @BeforeEach - public void setupTempFolder() throws IOException{ - folder = Files.createTempDirectory("junitTempFolder"); - } - - @AfterEach - public void teardownTempFolder() throws IOException{ - delete(folder); - } - - public Path createDirectory(String name) throws IOException { - Path newDirectory = folder.resolve(name); - return Files.createDirectories(newDirectory); - } - - public Path createFile(String name) throws IOException { - Path newFile = folder.resolve(name); - return Files.createFile(newFile); - } - - private void delete(Path tempDirectory) throws IOException { - Files.walkFileTree(tempDirectory, new SimpleFileVisitor() { - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - return deleteAndContinue(file); - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - return deleteAndContinue(dir); - } - - private FileVisitResult deleteAndContinue(Path path) throws IOException { - Files.delete(path); - return FileVisitResult.CONTINUE; - } - }); - } - -} diff --git a/src/test/java/gov/loc/repository/bagit/conformance/profile/BagitProfileDeserializerTest.java b/src/test/java/gov/loc/repository/bagit/conformance/profile/BagitProfileDeserializerTest.java deleted file mode 100644 index 1073a2f68..000000000 --- a/src/test/java/gov/loc/repository/bagit/conformance/profile/BagitProfileDeserializerTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package gov.loc.repository.bagit.conformance.profile; - -import java.io.File; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class BagitProfileDeserializerTest extends AbstractBagitProfileTest{ - - @Test - public void testDeserialize() throws Exception{ - BagitProfile expectedProfile = createExpectedProfile(); - - BagitProfile profile = mapper.readValue(new File("src/test/resources/bagitProfiles/exampleProfile.json"), BagitProfile.class); - - Assertions.assertEquals(expectedProfile, profile); - Assertions.assertEquals(expectedProfile.getAcceptableBagitVersions(), profile.getAcceptableBagitVersions()); - Assertions.assertEquals(expectedProfile.getAcceptableMIMESerializationTypes(), profile.getAcceptableMIMESerializationTypes()); - Assertions.assertEquals(expectedProfile.getBagInfoRequirements(), profile.getBagInfoRequirements()); - Assertions.assertEquals(expectedProfile.getBagitProfileIdentifier(), profile.getBagitProfileIdentifier()); - Assertions.assertEquals(expectedProfile.getContactEmail(), profile.getContactEmail()); - Assertions.assertEquals(expectedProfile.getContactName(), profile.getContactName()); - Assertions.assertEquals(expectedProfile.getExternalDescription(), profile.getExternalDescription()); - Assertions.assertEquals(expectedProfile.getManifestTypesRequired(), profile.getManifestTypesRequired()); - Assertions.assertEquals(expectedProfile.getSerialization(), profile.getSerialization()); - Assertions.assertEquals(expectedProfile.getSourceOrganization(), profile.getSourceOrganization()); - Assertions.assertEquals(expectedProfile.getTagFilesRequired(), profile.getTagFilesRequired()); - Assertions.assertEquals(expectedProfile.getTagManifestTypesRequired(), profile.getTagManifestTypesRequired()); - Assertions.assertEquals(expectedProfile.getVersion(), profile.getVersion()); - Assertions.assertEquals(expectedProfile.hashCode(), profile.hashCode()); - } - - -} diff --git a/src/test/java/gov/loc/repository/bagit/conformance/profile/BagitProfileTest.java b/src/test/java/gov/loc/repository/bagit/conformance/profile/BagitProfileTest.java deleted file mode 100644 index f9f844ce0..000000000 --- a/src/test/java/gov/loc/repository/bagit/conformance/profile/BagitProfileTest.java +++ /dev/null @@ -1,110 +0,0 @@ -package gov.loc.repository.bagit.conformance.profile; - -import java.io.File; -import java.util.Arrays; -import java.util.HashMap; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class BagitProfileTest extends AbstractBagitProfileTest{ - - @Test - public void testToString() throws Exception{ - String expectedOutput = "BagitProfile [bagitProfileIdentifier=http://canadiana.org/standards/bagit/tdr_ingest.json, " - + "sourceOrganization=Candiana.org, " - + "externalDescription=BagIt profile for ingesting content into the C.O. TDR loading dock., " - + "contactName=William Wueppelmann, " - + "contactEmail=tdr@canadiana.com, " - + "version=1.2, " - + "bagInfoRequirements={" - + "Payload-Oxum=[required=true, acceptableValues=[], repeatable=false], " - + "Bag-Size=[required=true, acceptableValues=[], repeatable=false], " - + "Bagging-Date=[required=true, acceptableValues=[], repeatable=false], " - + "Source-Organization=[required=true, acceptableValues=[Simon Fraser University, York University], repeatable=false], " - + "Bag-Count=[required=true, acceptableValues=[], repeatable=false], " - + "Organization-Address=[required=true, acceptableValues=[8888 University Drive Burnaby, B.C. V5A 1S6 Canada, 4700 Keele Street Toronto, Ontario M3J 1P3 Canada], repeatable=false], " - + "Bag-Group-Identifier=[required=false, acceptableValues=[], repeatable=false], " - + "External-Identifier=[required=false, acceptableValues=[], repeatable=false], " - + "Internal-Sender-Identifier=[required=false, acceptableValues=[], repeatable=false], " - + "Contact-Email=[required=true, acceptableValues=[], repeatable=false], " - + "Contact-Phone=[required=false, acceptableValues=[], repeatable=false], " - + "Internal-Sender-Description=[required=false, acceptableValues=[], repeatable=false], " - + "External-Description=[required=true, acceptableValues=[], repeatable=false], " - + "Contact-Name=[required=true, acceptableValues=[Mark Jordan, Nick Ruest], repeatable=false]}, " - + "manifestTypesRequired=[md5], " - + "fetchFileAllowed=false, " - + "serialization=forbidden, " - + "acceptableMIMESerializationTypes=[application/zip], " - + "acceptableBagitVersions=[0.96], " - + "tagManifestTypesRequired=[md5], " - + "tagFilesRequired=[DPN/dpnFirstNode.txt, DPN/dpnRegistry]]"; - - BagitProfile profile = mapper.readValue(new File("src/test/resources/bagitProfiles/exampleProfile.json"), BagitProfile.class); - System.err.println(profile.toString()); - Assertions.assertEquals(expectedOutput, profile.toString()); - } - - @Test - public void testEquals(){ - BagitProfile profile = createExpectedProfile(); - - Assertions.assertFalse(profile.equals(null)); - - BagitProfile differentBagitProfileIdentifier = createExpectedProfile(); - differentBagitProfileIdentifier.setBagitProfileIdentifier("foo"); - Assertions.assertFalse(profile.equals(differentBagitProfileIdentifier)); - - BagitProfile differentSourceOrganization = createExpectedProfile(); - differentSourceOrganization.setSourceOrganization("foo"); - Assertions.assertFalse(profile.equals(differentSourceOrganization)); - - BagitProfile differentExternalDescription = createExpectedProfile(); - differentExternalDescription.setExternalDescription("foo"); - Assertions.assertFalse(profile.equals(differentExternalDescription)); - - BagitProfile differentContactName = createExpectedProfile(); - differentContactName.setContactName("foo"); - Assertions.assertFalse(profile.equals(differentContactName)); - - BagitProfile differentContactEmail = createExpectedProfile(); - differentContactEmail.setContactEmail("foo"); - Assertions.assertFalse(profile.equals(differentContactEmail)); - - BagitProfile differentVersion = createExpectedProfile(); - differentVersion.setVersion("foo"); - Assertions.assertFalse(profile.equals(differentVersion)); - - BagitProfile differentBagInfoRequirements = createExpectedProfile(); - differentBagInfoRequirements.setBagInfoRequirements(new HashMap<>()); - Assertions.assertFalse(profile.equals(differentBagInfoRequirements)); - - BagitProfile differentManifestTypesRequired = createExpectedProfile(); - differentManifestTypesRequired.setManifestTypesRequired(Arrays.asList("foo")); - Assertions.assertFalse(profile.equals(differentManifestTypesRequired)); - - BagitProfile differentFetchFileAllowed = createExpectedProfile(); - differentFetchFileAllowed.setFetchFileAllowed(true); - Assertions.assertFalse(profile.equals(differentFetchFileAllowed)); - - BagitProfile differentSerialization = createExpectedProfile(); - differentSerialization.setSerialization(Serialization.required); - Assertions.assertFalse(profile.equals(differentSerialization)); - - BagitProfile differentAcceptableMIMESerializationTypes = createExpectedProfile(); - differentAcceptableMIMESerializationTypes.setAcceptableMIMESerializationTypes(Arrays.asList("foo")); - Assertions.assertFalse(profile.equals(differentAcceptableMIMESerializationTypes)); - - BagitProfile differentAcceptableBagitVersions = createExpectedProfile(); - differentAcceptableBagitVersions.setAcceptableBagitVersions(Arrays.asList("foo")); - Assertions.assertFalse(profile.equals(differentAcceptableBagitVersions)); - - BagitProfile differentTagManifestTypesRequired = createExpectedProfile(); - differentTagManifestTypesRequired.setTagManifestTypesRequired(Arrays.asList("foo")); - Assertions.assertFalse(profile.equals(differentTagManifestTypesRequired)); - - BagitProfile differentTagFilesRequired = createExpectedProfile(); - differentTagFilesRequired.setTagFilesRequired(Arrays.asList("foo")); - Assertions.assertFalse(profile.equals(differentTagFilesRequired)); - } -} diff --git a/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java b/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java deleted file mode 100644 index 303e0acae..000000000 --- a/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java +++ /dev/null @@ -1,156 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.Security; -import java.util.Arrays; -import java.util.List; - -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import gov.loc.repository.bagit.TempFolderTest; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.exceptions.CorruptChecksumException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.exceptions.VerificationException; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; -import gov.loc.repository.bagit.hash.SupportedAlgorithm; -import gov.loc.repository.bagit.reader.BagReader; - -public class BagVerifierTest extends TempFolderTest{ - static { - if (Security.getProvider("BC") == null) { - Security.addProvider(new BouncyCastleProvider()); - } - } - - private Path rootDir = Paths.get(new File("src/test/resources/bags/v0_97/bag").toURI()); - - private BagVerifier sut = new BagVerifier(); - private BagReader reader = new BagReader(); - - @Test - public void testStandardSupportedAlgorithms() throws Exception{ - List algorithms = Arrays.asList("md5", "sha1", "sha256", "sha512"); - for(String alg : algorithms){ - StandardSupportedAlgorithms algorithm = StandardSupportedAlgorithms.valueOf(alg.toUpperCase()); - Manifest manifest = new Manifest(algorithm); - sut.checkHashes(manifest); - } - } - - @Test - public void testVersion0_97IsValid() throws Exception{ - Bag bag = reader.read(rootDir); - - sut.isValid(bag, true); - } - - @Test - public void testVersion2_0IsValid() throws Exception{ - rootDir = Paths.get(new File("src/test/resources/bags/v2_0/bag").toURI()); - Bag bag = reader.read(rootDir); - - sut.isValid(bag, true); - } - - @Test - public void testIsComplete() throws Exception{ - Bag bag = reader.read(rootDir); - - sut.isComplete(bag, true); - } - - @Test - public void testCorruptPayloadFile() throws Exception{ - rootDir = Paths.get(new File("src/test/resources/corruptPayloadFile").toURI()); - Bag bag = reader.read(rootDir); - - Assertions.assertThrows(CorruptChecksumException.class, () -> { sut.isValid(bag, true); }); - } - - @Test - public void testCorruptTagFile() throws Exception{ - rootDir = Paths.get(new File("src/test/resources/corruptTagFile").toURI()); - Bag bag = reader.read(rootDir); - - Assertions.assertThrows(CorruptChecksumException.class, () -> { sut.isValid(bag, true); }); - } - - @Test - public void testErrorWhenUnspportedAlgorithmException() throws Exception{ - Path sha3BagDir = Paths.get(getClass().getClassLoader().getResource("sha3Bag").toURI()); - MySupportedNameToAlgorithmMapping mapping = new MySupportedNameToAlgorithmMapping(); - BagReader extendedReader = new BagReader(mapping); - Bag bag = extendedReader.read(sha3BagDir); - - Assertions.assertThrows(UnsupportedAlgorithmException.class, () -> { sut.isValid(bag, true); }); - } - - @Test - public void testVerificationExceptionIsThrownForNoSuchAlgorithmException() throws Exception{ - Path unreadableFile = createFile("newFile"); - - Manifest manifest = new Manifest(new SupportedAlgorithm() { - @Override - public String getMessageDigestName() { - return "FOO"; - } - @Override - public String getBagitName() { - return "foo"; - } - }); - manifest.getFileToChecksumMap().put(unreadableFile, "foo"); - - Assertions.assertThrows(VerificationException.class, () -> { sut.checkHashes(manifest); }); - } - - @Test - public void testAddSHA3SupportViaExtension() throws Exception{ - Path sha3BagDir = Paths.get(new File("src/test/resources/sha3Bag").toURI()); - MySupportedNameToAlgorithmMapping mapping = new MySupportedNameToAlgorithmMapping(); - BagReader extendedReader = new BagReader(mapping); - Bag bag = extendedReader.read(sha3BagDir); - try(BagVerifier extendedSut = new BagVerifier(mapping)){ - extendedSut.isValid(bag, true); - } - } - - /* - * Technically valid but highly discouraged - */ - @Test - public void testManifestsWithLeadingDotSlash() throws Exception{ - Path bagPath = Paths.get(new File("src/test/resources/bag-with-leading-dot-slash-in-manifest").toURI()); - Bag bag = reader.read(bagPath); - - sut.isValid(bag, true); - } - - @Test - public void testCanQuickVerify() throws Exception{ - Bag bag = reader.read(rootDir); - boolean canQuickVerify = BagVerifier.canQuickVerify(bag); - Assertions.assertFalse(canQuickVerify, - "Since " + bag.getRootDir() + " DOES NOT contain the metadata Payload-Oxum then it should return false!"); - - Path passingRootDir = Paths.get(new File("src/test/resources/bags/v0_94/bag").toURI()); - bag = reader.read(passingRootDir); - canQuickVerify = BagVerifier.canQuickVerify(bag); - Assertions.assertTrue(canQuickVerify, - "Since " + bag.getRootDir() + " DOES contain the metadata Payload-Oxum then it should return true!"); - } - - @Test - public void testQuickVerify() throws Exception{ - Path passingRootDir = Paths.get(new File("src/test/resources/bags/v0_94/bag").toURI()); - Bag bag = reader.read(passingRootDir); - - BagVerifier.quicklyVerify(bag); - } -} diff --git a/src/test/java/gov/loc/repository/bagit/verify/MySupportedNameToAlgorithmMapping.java b/src/test/java/gov/loc/repository/bagit/verify/MySupportedNameToAlgorithmMapping.java deleted file mode 100644 index 55e72898d..000000000 --- a/src/test/java/gov/loc/repository/bagit/verify/MySupportedNameToAlgorithmMapping.java +++ /dev/null @@ -1,18 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import gov.loc.repository.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; -import gov.loc.repository.bagit.hash.SupportedAlgorithm; - -public class MySupportedNameToAlgorithmMapping implements BagitAlgorithmNameToSupportedAlgorithmMapping { - - @Override - public SupportedAlgorithm getSupportedAlgorithm(String bagitAlgorithmName) { - if("sha3256".equals(bagitAlgorithmName)){ - return new SHA3256Algorithm(); - } - - return StandardSupportedAlgorithms.valueOf(bagitAlgorithmName.toUpperCase()); - } - -} diff --git a/src/test/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistorTest.java b/src/test/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistorTest.java deleted file mode 100644 index 41b2a74a1..000000000 --- a/src/test/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistorTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import java.util.HashSet; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import gov.loc.repository.bagit.TempFolderTest; -import gov.loc.repository.bagit.exceptions.FileNotInManifestException; - -public class PayloadFileExistsInAtLeastOneManifestVistorTest extends TempFolderTest { - - @Test - public void testFileNotInManifestException() throws Exception{ - - PayloadFileExistsInAtLeastOneManifestVistor sut = new PayloadFileExistsInAtLeastOneManifestVistor(new HashSet<>(), true); - Assertions.assertThrows(FileNotInManifestException.class, - () -> { sut.visitFile(createFile("aNewFile"), null); }); - } -} diff --git a/src/test/java/gov/loc/repository/bagit/verify/SHA3256Algorithm.java b/src/test/java/gov/loc/repository/bagit/verify/SHA3256Algorithm.java deleted file mode 100644 index 6f18bdb65..000000000 --- a/src/test/java/gov/loc/repository/bagit/verify/SHA3256Algorithm.java +++ /dev/null @@ -1,17 +0,0 @@ -package gov.loc.repository.bagit.verify; - -import gov.loc.repository.bagit.hash.SupportedAlgorithm; - -public class SHA3256Algorithm implements SupportedAlgorithm { - - @Override - public String getMessageDigestName() { - return "SHA3-256"; - } - - @Override - public String getBagitName() { - return "sha3256"; - } - -} diff --git a/src/test/java/gov/loc/repository/bagit/writer/MetadataWriterTest.java b/src/test/java/gov/loc/repository/bagit/writer/MetadataWriterTest.java deleted file mode 100644 index f6ee0368f..000000000 --- a/src/test/java/gov/loc/repository/bagit/writer/MetadataWriterTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package gov.loc.repository.bagit.writer; - -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Metadata; -import gov.loc.repository.bagit.domain.Version; - -public class MetadataWriterTest extends PrivateConstructorTest { - - @Test - public void testClassIsWellDefined() throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException{ - assertUtilityClassWellDefined(MetadataWriter.class); - } - - @Test - public void testWriteBagitInfoFile() throws IOException{ - Path rootDir = createDirectory("writeBagitInfo"); - Path bagInfo = rootDir.resolve("bag-info.txt"); - Path packageInfo = rootDir.resolve("package-info.txt"); - Metadata metadata = new Metadata(); - metadata.add("key1", "value1"); - metadata.add("key2", "value2"); - metadata.add("key3", "value3"); - - Assertions.assertFalse(Files.exists(bagInfo)); - Assertions.assertFalse(Files.exists(packageInfo)); - - MetadataWriter.writeBagMetadata(metadata, new Version(0,96), rootDir, StandardCharsets.UTF_8); - Assertions.assertTrue(Files.exists(bagInfo)); - - MetadataWriter.writeBagMetadata(metadata, new Version(0,95), rootDir, StandardCharsets.UTF_8); - Assertions.assertTrue(Files.exists(packageInfo)); - } -} diff --git a/src/test/java/gov/loc/repository/bagit/PrivateConstructorTest.java b/src/test/java/nl/knaw/dans/bagit/PrivateConstructorTest.java similarity index 68% rename from src/test/java/gov/loc/repository/bagit/PrivateConstructorTest.java rename to src/test/java/nl/knaw/dans/bagit/PrivateConstructorTest.java index e78907d48..c63a09ba8 100644 --- a/src/test/java/gov/loc/repository/bagit/PrivateConstructorTest.java +++ b/src/test/java/nl/knaw/dans/bagit/PrivateConstructorTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/src/test/java/nl/knaw/dans/bagit/TempFolderTest.java b/src/test/java/nl/knaw/dans/bagit/TempFolderTest.java new file mode 100644 index 000000000..dea3fe640 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/TempFolderTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; + +abstract public class TempFolderTest { + protected Path folder; + + @BeforeEach + public void setupTempFolder() throws IOException{ + folder = Files.createTempDirectory("junitTempFolder"); + } + + @AfterEach + public void teardownTempFolder() throws IOException{ + delete(folder); + Assertions.assertFalse(Files.exists(folder)); + //Assertions.assertEquals(0, Files.list(folder).count()); + } + + public Path createDirectory(String name) throws IOException { + Path newDirectory = folder.resolve(name); + return Files.createDirectories(newDirectory); + } + + public Path createFile(String name) throws IOException { + Path newFile = folder.resolve(name); + return Files.createFile(newFile); + } + + public Path copyBagToTempFolder(Path bagFolder) throws IOException{ + Path bagCopyDir = createDirectory(bagFolder.getFileName() + "_copy"); + Files.walkFileTree(bagFolder, new SimpleFileVisitor() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path relative = bagFolder.relativize(file); + if(relative.getParent() != null) { + Files.createDirectories(bagCopyDir.resolve(relative.getParent())); + } + Files.copy(file, bagCopyDir.resolve(relative)); + return FileVisitResult.CONTINUE; + } + }); + + return bagCopyDir; + } + + protected void delete(Path tempDirectory) throws IOException { + Files.walkFileTree(tempDirectory, new SimpleFileVisitor() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + return deleteAndContinue(file); + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + return deleteAndContinue(dir); + } + + private FileVisitResult deleteAndContinue(Path path) throws IOException { + Files.delete(path); + return FileVisitResult.CONTINUE; + } + }); + } + +} diff --git a/src/test/java/gov/loc/repository/bagit/TestUtils.java b/src/test/java/nl/knaw/dans/bagit/TestUtils.java similarity index 59% rename from src/test/java/gov/loc/repository/bagit/TestUtils.java rename to src/test/java/nl/knaw/dans/bagit/TestUtils.java index 619760301..22a153d85 100644 --- a/src/test/java/gov/loc/repository/bagit/TestUtils.java +++ b/src/test/java/nl/knaw/dans/bagit/TestUtils.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -12,6 +27,12 @@ public static boolean isExecutingOnWindows(){ return System.getProperty("os.name").contains("Windows"); } + /** + * walk a directory and make sure that files/folders are hidden if they start with a . on windows. + * + * @param startingDir the directory to start walking + * @throws IOException if there is a problem setting the file/folder to be hidden + */ public static void makeFilesHiddenOnWindows(Path startingDir) throws IOException { if (isExecutingOnWindows()) { Files.walkFileTree(startingDir, new SimpleFileVisitor() { diff --git a/src/test/java/gov/loc/repository/bagit/conformance/BagLinterTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/BagLinterTest.java similarity index 64% rename from src/test/java/gov/loc/repository/bagit/conformance/BagLinterTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/BagLinterTest.java index 3a9655fbb..f04d8c2fa 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/BagLinterTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/BagLinterTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.io.File; import java.io.InputStream; @@ -12,14 +27,14 @@ import java.util.HashSet; import java.util.Set; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.Bag; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.bagit.reader.BagReader; -public class BagLinterTest extends PrivateConstructorTest{ +public class BagLinterTest extends PrivateConstructorTest { private final Path rootDir = Paths.get("src","test","resources","linterTestBag"); @@ -28,6 +43,13 @@ public void testClassIsWellDefined() throws NoSuchMethodException, InvocationTar assertUtilityClassWellDefined(BagLinter.class); } + @Test + public void testConformantBag() throws Exception{ + Path goodBag = Paths.get("src", "test", "resources", "bags", "v1_0", "bag"); + Set warnings = BagLinter.lintBag(goodBag); + Assertions.assertTrue(warnings.size() == 0); + } + @Test public void testLintBag() throws Exception{ Set expectedWarnings = new HashSet<>(); @@ -35,8 +57,9 @@ public void testLintBag() throws Exception{ expectedWarnings.remove(BagitWarning.MANIFEST_SETS_DIFFER); //only applies to version 1.0 but need older version for other warnings, so we test this separately Set warnings = BagLinter.lintBag(rootDir); - if(FileSystems.getDefault().getClass().getName() == "sun.nio.fs.MacOSXFileSystem"){ - expectedWarnings.remove(BagitWarning.DIFFERENT_NORMALIZATION); //don't test normalization on mac + if(FileSystems.getDefault().getClass().getName() == "sun.nio.fs.MacOSXFileSystem"){ //don't test normalization on mac + expectedWarnings.remove(BagitWarning.DIFFERENT_NORMALIZATION); + warnings.remove(BagitWarning.DIFFERENT_NORMALIZATION); } Set diff = new HashSet<>(expectedWarnings); diff --git a/src/test/java/gov/loc/repository/bagit/conformance/BagProfileCheckerTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/BagProfileCheckerTest.java similarity index 78% rename from src/test/java/gov/loc/repository/bagit/conformance/BagProfileCheckerTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/BagProfileCheckerTest.java index 1074d2da4..c17d8ef2f 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/BagProfileCheckerTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/BagProfileCheckerTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.io.File; import java.io.InputStream; @@ -7,19 +22,19 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.Bag; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.exceptions.conformance.BagitVersionIsNotAcceptableException; -import gov.loc.repository.bagit.exceptions.conformance.FetchFileNotAllowedException; -import gov.loc.repository.bagit.exceptions.conformance.MetatdataValueIsNotAcceptableException; -import gov.loc.repository.bagit.exceptions.conformance.MetatdataValueIsNotRepeatableException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredManifestNotPresentException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredMetadataFieldNotPresentException; -import gov.loc.repository.bagit.exceptions.conformance.RequiredTagFileNotPresentException; -import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.bagit.exceptions.conformance.BagitVersionIsNotAcceptableException; +import nl.knaw.dans.bagit.exceptions.conformance.FetchFileNotAllowedException; +import nl.knaw.dans.bagit.exceptions.conformance.MetatdataValueIsNotAcceptableException; +import nl.knaw.dans.bagit.exceptions.conformance.MetatdataValueIsNotRepeatableException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredManifestNotPresentException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredMetadataFieldNotPresentException; +import nl.knaw.dans.bagit.exceptions.conformance.RequiredTagFileNotPresentException; +import nl.knaw.dans.bagit.reader.BagReader; public class BagProfileCheckerTest extends PrivateConstructorTest { private static final Path profileJson = new File("src/test/resources/bagitProfiles/exampleProfile.json").toPath(); diff --git a/src/test/java/gov/loc/repository/bagit/conformance/EncodingCheckerTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/EncodingCheckerTest.java similarity index 55% rename from src/test/java/gov/loc/repository/bagit/conformance/EncodingCheckerTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/EncodingCheckerTest.java index 1f21a1f53..2ae22006a 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/EncodingCheckerTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/EncodingCheckerTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.nio.charset.StandardCharsets; import java.util.Arrays; diff --git a/src/test/java/gov/loc/repository/bagit/conformance/ManifestCheckerTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/ManifestCheckerTest.java similarity index 87% rename from src/test/java/gov/loc/repository/bagit/conformance/ManifestCheckerTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/ManifestCheckerTest.java index 537fe812a..2fe809be1 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/ManifestCheckerTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/ManifestCheckerTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; @@ -10,14 +25,14 @@ import java.util.HashSet; import java.util.Set; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.domain.Version; -public class ManifestCheckerTest extends PrivateConstructorTest{ +public class ManifestCheckerTest extends PrivateConstructorTest { private final Path rootDir = Paths.get("src","test","resources","linterTestBag"); @@ -146,7 +161,7 @@ public void testOSSpecificFilesRegex(){ } @Test - public void testParsePath() throws InvalidBagitFileFormatException{ + public void testParsePath() throws InvalidBagitFileFormatException { Assertions.assertThrows(InvalidBagitFileFormatException.class, () -> { ManifestChecker.parsePath("foobarham"); }); } diff --git a/src/test/java/gov/loc/repository/bagit/conformance/MetadataCheckerTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/MetadataCheckerTest.java similarity index 64% rename from src/test/java/gov/loc/repository/bagit/conformance/MetadataCheckerTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/MetadataCheckerTest.java index 822e660a0..7bb28e03e 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/MetadataCheckerTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/MetadataCheckerTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; @@ -9,11 +24,10 @@ import java.util.HashSet; import java.util.Set; +import nl.knaw.dans.bagit.PrivateConstructorTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; - public class MetadataCheckerTest extends PrivateConstructorTest { private final Path rootDir = Paths.get("src","test","resources","linterTestBag"); diff --git a/src/test/java/gov/loc/repository/bagit/conformance/VersionCheckerTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/VersionCheckerTest.java similarity index 52% rename from src/test/java/gov/loc/repository/bagit/conformance/VersionCheckerTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/VersionCheckerTest.java index b5450cde6..e0bcd9c6c 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/VersionCheckerTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/VersionCheckerTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance; import java.util.Arrays; import java.util.Collections; @@ -8,7 +23,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.domain.Version; +import nl.knaw.dans.bagit.domain.Version; public class VersionCheckerTest { diff --git a/src/test/java/gov/loc/repository/bagit/conformance/profile/AbstractBagitProfileTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/profile/AbstractBagitProfileTest.java similarity index 58% rename from src/test/java/gov/loc/repository/bagit/conformance/profile/AbstractBagitProfileTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/profile/AbstractBagitProfileTest.java index 3bba614eb..eb98c1760 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/profile/AbstractBagitProfileTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/profile/AbstractBagitProfileTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance.profile; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; import java.util.Arrays; import java.util.HashMap; @@ -26,6 +41,7 @@ protected BagitProfile createExpectedProfile(){ expectedProfile.setBagitProfileIdentifier("http://canadiana.org/standards/bagit/tdr_ingest.json"); expectedProfile.setContactEmail("tdr@canadiana.com"); expectedProfile.setContactName("William Wueppelmann"); + expectedProfile.setContactPhone("+1 613 907 7040"); expectedProfile.setExternalDescription("BagIt profile for ingesting content into the C.O. TDR loading dock."); expectedProfile.setSourceOrganization("Candiana.org"); expectedProfile.setVersion("1.2"); @@ -42,24 +58,42 @@ protected BagitProfile createExpectedProfile(){ return expectedProfile; } + protected BagitProfile createMinimalProfile(){ + BagitProfile expectedProfile = new BagitProfile(); + + expectedProfile.setBagitProfileIdentifier("http://canadiana.org/standards/bagit/tdr_ingest.json"); + expectedProfile.setExternalDescription("BagIt profile for ingesting content into the C.O. TDR loading dock."); + expectedProfile.setSourceOrganization("Candiana.org"); + expectedProfile.setVersion("1.2"); + + expectedProfile.setBagInfoRequirements(createBagInfo()); + expectedProfile.setManifestTypesRequired(Arrays.asList("md5")); + expectedProfile.setFetchFileAllowed(false); + expectedProfile.setSerialization(Serialization.forbidden); + expectedProfile.setAcceptableMIMESerializationTypes(Arrays.asList("application/zip")); + expectedProfile.setAcceptableBagitVersions(Arrays.asList("0.96")); + + return expectedProfile; + } + protected Map createBagInfo(){ Map info = new HashMap<>(); - info.put("Source-Organization", new BagInfoRequirement(true, Arrays.asList("Simon Fraser University", "York University"))); + info.put("Source-Organization", new BagInfoRequirement(true, Arrays.asList("Simon Fraser University", "York University"), false)); info.put("Organization-Address", new BagInfoRequirement(true, - Arrays.asList("8888 University Drive Burnaby, B.C. V5A 1S6 Canada", "4700 Keele Street Toronto, Ontario M3J 1P3 Canada"))); - info.put("Contact-Name", new BagInfoRequirement(true, Arrays.asList("Mark Jordan", "Nick Ruest"))); - info.put("Contact-Phone", new BagInfoRequirement(false, Arrays.asList())); - info.put("Contact-Email", new BagInfoRequirement(true, Arrays.asList())); - info.put("External-Description", new BagInfoRequirement(true, Arrays.asList())); - info.put("External-Identifier", new BagInfoRequirement(false, Arrays.asList())); - info.put("Bag-Size", new BagInfoRequirement(true, Arrays.asList())); - info.put("Bag-Group-Identifier", new BagInfoRequirement(false, Arrays.asList())); - info.put("Bag-Count", new BagInfoRequirement(true, Arrays.asList())); - info.put("Internal-Sender-Identifier", new BagInfoRequirement(false, Arrays.asList())); - info.put("Internal-Sender-Description", new BagInfoRequirement(false, Arrays.asList())); - info.put("Bagging-Date", new BagInfoRequirement(true, Arrays.asList())); - info.put("Payload-Oxum", new BagInfoRequirement(true, Arrays.asList())); + Arrays.asList("8888 University Drive Burnaby, B.C. V5A 1S6 Canada", "4700 Keele Street Toronto, Ontario M3J 1P3 Canada"), false)); + info.put("Contact-Name", new BagInfoRequirement(true, Arrays.asList("Mark Jordan", "Nick Ruest"), false)); + info.put("Contact-Phone", new BagInfoRequirement(false, Arrays.asList(), false)); + info.put("Contact-Email", new BagInfoRequirement(true, Arrays.asList(), false)); + info.put("External-Description", new BagInfoRequirement(true, Arrays.asList(), false)); + info.put("External-Identifier", new BagInfoRequirement(false, Arrays.asList(), false)); + info.put("Bag-Size", new BagInfoRequirement(true, Arrays.asList(), false)); + info.put("Bag-Group-Identifier", new BagInfoRequirement(false, Arrays.asList(), false)); + info.put("Bag-Count", new BagInfoRequirement(true, Arrays.asList(), false)); + info.put("Internal-Sender-Identifier", new BagInfoRequirement(false, Arrays.asList(), false)); + info.put("Internal-Sender-Description", new BagInfoRequirement(false, Arrays.asList(), false)); + info.put("Bagging-Date", new BagInfoRequirement(true, Arrays.asList(), false)); + info.put("Payload-Oxum", new BagInfoRequirement(true, Arrays.asList(), false)); return info; } diff --git a/src/test/java/gov/loc/repository/bagit/conformance/profile/BagInfoRequirementTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagInfoRequirementTest.java similarity index 59% rename from src/test/java/gov/loc/repository/bagit/conformance/profile/BagInfoRequirementTest.java rename to src/test/java/nl/knaw/dans/bagit/conformance/profile/BagInfoRequirementTest.java index 14bfb9f63..35c3bd87c 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/profile/BagInfoRequirementTest.java +++ b/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagInfoRequirementTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.conformance.profile; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; import java.util.Arrays; diff --git a/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileDeserializerTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileDeserializerTest.java new file mode 100644 index 000000000..a1fa94fd5 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileDeserializerTest.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; + +import java.io.File; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class BagitProfileDeserializerTest extends AbstractBagitProfileTest{ + + @Test + public void testDeserialize() throws Exception{ + BagitProfile expectedProfile = createExpectedProfile(); + + BagitProfile profile = mapper.readValue(new File("src/test/resources/bagitProfiles/exampleProfile.json"), BagitProfile.class); + + Assertions.assertEquals(expectedProfile, profile); + Assertions.assertEquals(expectedProfile.getAcceptableBagitVersions(), profile.getAcceptableBagitVersions()); + Assertions.assertEquals(expectedProfile.getAcceptableMIMESerializationTypes(), profile.getAcceptableMIMESerializationTypes()); + Assertions.assertEquals(expectedProfile.getBagInfoRequirements(), profile.getBagInfoRequirements()); + Assertions.assertEquals(expectedProfile.getBagitProfileIdentifier(), profile.getBagitProfileIdentifier()); + Assertions.assertEquals(expectedProfile.getContactEmail(), profile.getContactEmail()); + Assertions.assertEquals(expectedProfile.getContactName(), profile.getContactName()); + Assertions.assertEquals(expectedProfile.getContactPhone(), profile.getContactPhone()); + Assertions.assertEquals(expectedProfile.getExternalDescription(), profile.getExternalDescription()); + Assertions.assertEquals(expectedProfile.getManifestTypesRequired(), profile.getManifestTypesRequired()); + Assertions.assertEquals(expectedProfile.getSerialization(), profile.getSerialization()); + Assertions.assertEquals(expectedProfile.getSourceOrganization(), profile.getSourceOrganization()); + Assertions.assertEquals(expectedProfile.getTagFilesRequired(), profile.getTagFilesRequired()); + Assertions.assertEquals(expectedProfile.getTagManifestTypesRequired(), profile.getTagManifestTypesRequired()); + Assertions.assertEquals(expectedProfile.getVersion(), profile.getVersion()); + Assertions.assertEquals(expectedProfile.hashCode(), profile.hashCode()); + } + + @Test + public void testDeserializeWithoutOptionalTags() throws Exception{ + BagitProfile minimalProfile = createMinimalProfile(); + + BagitProfile profile = mapper.readValue(new File("src/test/resources/bagitProfiles/exampleProfileOnlyRequiredFields.json"), BagitProfile.class); + Assertions.assertEquals(minimalProfile, profile); + Assertions.assertEquals(minimalProfile.getAcceptableBagitVersions(), profile.getAcceptableBagitVersions()); + Assertions.assertEquals(minimalProfile.getAcceptableMIMESerializationTypes(), profile.getAcceptableMIMESerializationTypes()); + Assertions.assertEquals(minimalProfile.getBagInfoRequirements(), profile.getBagInfoRequirements()); + Assertions.assertEquals(minimalProfile.getBagitProfileIdentifier(), profile.getBagitProfileIdentifier()); + Assertions.assertEquals(minimalProfile.getContactEmail(), profile.getContactEmail()); + Assertions.assertEquals(minimalProfile.getContactName(), profile.getContactName()); + Assertions.assertEquals(minimalProfile.getContactPhone(), profile.getContactPhone()); + Assertions.assertEquals(minimalProfile.getExternalDescription(), profile.getExternalDescription()); + Assertions.assertEquals(minimalProfile.getManifestTypesRequired(), profile.getManifestTypesRequired()); + Assertions.assertEquals(minimalProfile.getSerialization(), profile.getSerialization()); + Assertions.assertEquals(minimalProfile.getSourceOrganization(), profile.getSourceOrganization()); + Assertions.assertEquals(minimalProfile.getTagFilesRequired(), profile.getTagFilesRequired()); + Assertions.assertEquals(minimalProfile.getTagManifestTypesRequired(), profile.getTagManifestTypesRequired()); + Assertions.assertEquals(minimalProfile.getVersion(), profile.getVersion()); + Assertions.assertEquals(minimalProfile.hashCode(), profile.hashCode()); + } + + +} diff --git a/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileTest.java b/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileTest.java new file mode 100644 index 000000000..0eab58d05 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/conformance/profile/BagitProfileTest.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.conformance.profile; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.HashMap; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +public class BagitProfileTest extends AbstractBagitProfileTest { + + @Test + public void testToString() throws Exception { + String expectedOutput = "BagitProfile [bagitProfileIdentifier=http://canadiana.org/standards/bagit/tdr_ingest.json, " + + "sourceOrganization=Candiana.org, " + + "externalDescription=BagIt profile for ingesting content into the C.O. TDR loading dock., " + + "contactName=William Wueppelmann, " + + "contactEmail=tdr@canadiana.com, " + + "contactPhone=+1 613 907 7040, " + + "version=1.2, " + + "bagInfoRequirements={" + + "Payload-Oxum=[required=true, acceptableValues=[], repeatable=false], " + + "Bag-Size=[required=true, acceptableValues=[], repeatable=false], " + + "Bagging-Date=[required=true, acceptableValues=[], repeatable=false], " + + "Source-Organization=[required=true, acceptableValues=[Simon Fraser University, York University], repeatable=false], " + + "Bag-Count=[required=true, acceptableValues=[], repeatable=false], " + + "Organization-Address=[required=true, acceptableValues=[8888 University Drive Burnaby, B.C. V5A 1S6 Canada, 4700 Keele Street Toronto, Ontario M3J 1P3 Canada], repeatable=false], " + + "Bag-Group-Identifier=[required=false, acceptableValues=[], repeatable=false], " + + "External-Identifier=[required=false, acceptableValues=[], repeatable=false], " + + "Internal-Sender-Identifier=[required=false, acceptableValues=[], repeatable=false], " + + "Contact-Email=[required=true, acceptableValues=[], repeatable=false], " + + "Contact-Phone=[required=false, acceptableValues=[], repeatable=false], " + + "Internal-Sender-Description=[required=false, acceptableValues=[], repeatable=false], " + + "External-Description=[required=true, acceptableValues=[], repeatable=false], " + + "Contact-Name=[required=true, acceptableValues=[Mark Jordan, Nick Ruest], repeatable=false]}, " + + "manifestTypesRequired=[md5], " + + "fetchFileAllowed=false, " + + "serialization=forbidden, " + + "acceptableMIMESerializationTypes=[application/zip], " + + "acceptableBagitVersions=[0.96], " + + "tagManifestTypesRequired=[md5], " + + "tagFilesRequired=[DPN/dpnFirstNode.txt, DPN/dpnRegistry]]"; + + BagitProfile profile = mapper.readValue(new File("src/test/resources/bagitProfiles/exampleProfile.json"), BagitProfile.class); + Assertions.assertEquals(expectedOutput, profile.toString()); + } + + @Test + public void testEquals() { + BagitProfile profile = createExpectedProfile(); + + assertNotEquals(null, profile); + + BagitProfile differentBagitProfileIdentifier = createExpectedProfile(); + differentBagitProfileIdentifier.setBagitProfileIdentifier("foo"); + assertNotEquals(profile, differentBagitProfileIdentifier); + + BagitProfile differentSourceOrganization = createExpectedProfile(); + differentSourceOrganization.setSourceOrganization("foo"); + assertNotEquals(profile, differentSourceOrganization); + + BagitProfile differentExternalDescription = createExpectedProfile(); + differentExternalDescription.setExternalDescription("foo"); + assertNotEquals(profile, differentExternalDescription); + + BagitProfile differentContactName = createExpectedProfile(); + differentContactName.setContactName("foo"); + assertNotEquals(profile, differentContactName); + + BagitProfile differentContactEmail = createExpectedProfile(); + differentContactEmail.setContactEmail("foo"); + assertNotEquals(profile, differentContactEmail); + + BagitProfile differentContactPhone = createExpectedProfile(); + differentContactPhone.setContactPhone("foo"); + assertNotEquals(profile, differentContactPhone); + + BagitProfile differentVersion = createExpectedProfile(); + differentVersion.setVersion("foo"); + assertNotEquals(profile, differentVersion); + + BagitProfile differentBagInfoRequirements = createExpectedProfile(); + differentBagInfoRequirements.setBagInfoRequirements(new HashMap<>()); + assertNotEquals(profile, differentBagInfoRequirements); + + BagitProfile differentManifestTypesRequired = createExpectedProfile(); + differentManifestTypesRequired.setManifestTypesRequired(List.of("foo")); + assertNotEquals(profile, differentManifestTypesRequired); + + BagitProfile differentFetchFileAllowed = createExpectedProfile(); + differentFetchFileAllowed.setFetchFileAllowed(true); + assertNotEquals(profile, differentFetchFileAllowed); + + BagitProfile differentSerialization = createExpectedProfile(); + differentSerialization.setSerialization(Serialization.required); + assertNotEquals(profile, differentSerialization); + + BagitProfile differentAcceptableMIMESerializationTypes = createExpectedProfile(); + differentAcceptableMIMESerializationTypes.setAcceptableMIMESerializationTypes(List.of("foo")); + assertNotEquals(profile, differentAcceptableMIMESerializationTypes); + + BagitProfile differentAcceptableBagitVersions = createExpectedProfile(); + differentAcceptableBagitVersions.setAcceptableBagitVersions(List.of("foo")); + assertNotEquals(profile, differentAcceptableBagitVersions); + + BagitProfile differentTagManifestTypesRequired = createExpectedProfile(); + differentTagManifestTypesRequired.setTagManifestTypesRequired(List.of("foo")); + assertNotEquals(profile, differentTagManifestTypesRequired); + + BagitProfile differentTagFilesRequired = createExpectedProfile(); + differentTagFilesRequired.setTagFilesRequired(List.of("foo")); + assertNotEquals(profile, differentTagFilesRequired); + } +} diff --git a/src/test/java/gov/loc/repository/bagit/creator/AddPayloadToBagManifestVistorTest.java b/src/test/java/nl/knaw/dans/bagit/creator/AddPayloadToBagManifestVistorTest.java similarity index 78% rename from src/test/java/gov/loc/repository/bagit/creator/AddPayloadToBagManifestVistorTest.java rename to src/test/java/nl/knaw/dans/bagit/creator/AddPayloadToBagManifestVistorTest.java index 09eb29bc1..3bdd2db1f 100644 --- a/src/test/java/gov/loc/repository/bagit/creator/AddPayloadToBagManifestVistorTest.java +++ b/src/test/java/nl/knaw/dans/bagit/creator/AddPayloadToBagManifestVistorTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.creator; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.creator; import java.io.File; import java.io.IOException; @@ -10,14 +25,13 @@ import java.util.HashMap; import java.util.Map; +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.TestUtils; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.TempFolderTest; -import gov.loc.repository.bagit.TestUtils; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; - public class AddPayloadToBagManifestVistorTest extends TempFolderTest { @Test diff --git a/src/test/java/gov/loc/repository/bagit/creator/BagCreatorTest.java b/src/test/java/nl/knaw/dans/bagit/creator/BagCreatorTest.java similarity index 84% rename from src/test/java/gov/loc/repository/bagit/creator/BagCreatorTest.java rename to src/test/java/nl/knaw/dans/bagit/creator/BagCreatorTest.java index b8432f9a0..20e71d4fc 100644 --- a/src/test/java/gov/loc/repository/bagit/creator/BagCreatorTest.java +++ b/src/test/java/nl/knaw/dans/bagit/creator/BagCreatorTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.creator; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.creator; import java.io.File; import java.io.IOException; @@ -9,16 +24,16 @@ import java.util.Arrays; import java.util.List; +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.TestUtils; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; +import nl.knaw.dans.bagit.util.PathUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.TempFolderTest; -import gov.loc.repository.bagit.TestUtils; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; -import gov.loc.repository.bagit.util.PathUtils; +import nl.knaw.dans.bagit.domain.Version; public class BagCreatorTest extends TempFolderTest { @@ -102,7 +117,7 @@ private TestStructure createTestStructure() throws IOException{ Assertions.assertTrue(Files.isHidden(hiddenFile)); //because the Files.isHidden() always returns false for windows if it is a directory - Assertions.assertTrue(PathUtils.isHidden(hiddenDirectory)); + Assertions.assertTrue(PathUtils.isHidden(hiddenDirectory)); Path hiddenFile2 = hiddenDirectory.resolve(".hiddenFile2.txt"); Files.createFile(hiddenFile2); diff --git a/src/test/java/gov/loc/repository/bagit/domain/BagTest.java b/src/test/java/nl/knaw/dans/bagit/domain/BagTest.java similarity index 72% rename from src/test/java/gov/loc/repository/bagit/domain/BagTest.java rename to src/test/java/nl/knaw/dans/bagit/domain/BagTest.java index a86841bbc..6a87a00e8 100644 --- a/src/test/java/gov/loc/repository/bagit/domain/BagTest.java +++ b/src/test/java/nl/knaw/dans/bagit/domain/BagTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.net.MalformedURLException; import java.net.URL; @@ -7,7 +22,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; public class BagTest { diff --git a/src/test/java/gov/loc/repository/bagit/domain/FetchItemTest.java b/src/test/java/nl/knaw/dans/bagit/domain/FetchItemTest.java similarity index 77% rename from src/test/java/gov/loc/repository/bagit/domain/FetchItemTest.java rename to src/test/java/nl/knaw/dans/bagit/domain/FetchItemTest.java index 2101a0d55..54fa4a7b9 100644 --- a/src/test/java/gov/loc/repository/bagit/domain/FetchItemTest.java +++ b/src/test/java/nl/knaw/dans/bagit/domain/FetchItemTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.io.File; import java.net.MalformedURLException; diff --git a/src/test/java/gov/loc/repository/bagit/domain/ManifestTest.java b/src/test/java/nl/knaw/dans/bagit/domain/ManifestTest.java similarity index 72% rename from src/test/java/gov/loc/repository/bagit/domain/ManifestTest.java rename to src/test/java/nl/knaw/dans/bagit/domain/ManifestTest.java index 69f1edcf6..ae5a1a563 100644 --- a/src/test/java/gov/loc/repository/bagit/domain/ManifestTest.java +++ b/src/test/java/nl/knaw/dans/bagit/domain/ManifestTest.java @@ -1,9 +1,24 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; public class ManifestTest { diff --git a/src/test/java/gov/loc/repository/bagit/domain/MetadataTest.java b/src/test/java/nl/knaw/dans/bagit/domain/MetadataTest.java similarity index 81% rename from src/test/java/gov/loc/repository/bagit/domain/MetadataTest.java rename to src/test/java/nl/knaw/dans/bagit/domain/MetadataTest.java index e1ed19ac7..ea7c1fce4 100644 --- a/src/test/java/gov/loc/repository/bagit/domain/MetadataTest.java +++ b/src/test/java/nl/knaw/dans/bagit/domain/MetadataTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import java.util.Arrays; diff --git a/src/test/java/gov/loc/repository/bagit/domain/VersionTest.java b/src/test/java/nl/knaw/dans/bagit/domain/VersionTest.java similarity index 84% rename from src/test/java/gov/loc/repository/bagit/domain/VersionTest.java rename to src/test/java/nl/knaw/dans/bagit/domain/VersionTest.java index c97ea48db..ac41ab7f1 100644 --- a/src/test/java/gov/loc/repository/bagit/domain/VersionTest.java +++ b/src/test/java/nl/knaw/dans/bagit/domain/VersionTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.domain; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.domain; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/src/test/java/gov/loc/repository/bagit/examples/fetching/FetchHttpFileExample.java b/src/test/java/nl/knaw/dans/bagit/examples/fetching/FetchHttpFileExample.java similarity index 50% rename from src/test/java/gov/loc/repository/bagit/examples/fetching/FetchHttpFileExample.java rename to src/test/java/nl/knaw/dans/bagit/examples/fetching/FetchHttpFileExample.java index cf4935851..2b38c6895 100644 --- a/src/test/java/gov/loc/repository/bagit/examples/fetching/FetchHttpFileExample.java +++ b/src/test/java/nl/knaw/dans/bagit/examples/fetching/FetchHttpFileExample.java @@ -1,17 +1,31 @@ -package gov.loc.repository.bagit.examples.fetching; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.examples.fetching; import java.io.IOException; import java.net.URL; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.domain.FetchItem; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.TempFolderTest; -import gov.loc.repository.bagit.domain.FetchItem; - -public class FetchHttpFileExample extends TempFolderTest{ +public class FetchHttpFileExample extends TempFolderTest { /** * THIS IS JUST AN EXAMPLE. DO NOT USE IN PRODUCTION! diff --git a/src/test/java/gov/loc/repository/bagit/examples/serialization/CreateTarBagExample.java b/src/test/java/nl/knaw/dans/bagit/examples/serialization/CreateTarBagExample.java similarity index 73% rename from src/test/java/gov/loc/repository/bagit/examples/serialization/CreateTarBagExample.java rename to src/test/java/nl/knaw/dans/bagit/examples/serialization/CreateTarBagExample.java index 31d49d436..2626fe3f6 100644 --- a/src/test/java/gov/loc/repository/bagit/examples/serialization/CreateTarBagExample.java +++ b/src/test/java/nl/knaw/dans/bagit/examples/serialization/CreateTarBagExample.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.examples.serialization; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.examples.serialization; import java.io.File; import java.io.IOException; @@ -11,14 +26,13 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; +import nl.knaw.dans.bagit.TempFolderTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.kamranzafar.jtar.TarEntry; import org.kamranzafar.jtar.TarOutputStream; -import gov.loc.repository.bagit.TempFolderTest; - public class CreateTarBagExample extends TempFolderTest { private Path bagRoot; diff --git a/src/test/java/gov/loc/repository/bagit/examples/serialization/CreateZipBagExample.java b/src/test/java/nl/knaw/dans/bagit/examples/serialization/CreateZipBagExample.java similarity index 72% rename from src/test/java/gov/loc/repository/bagit/examples/serialization/CreateZipBagExample.java rename to src/test/java/nl/knaw/dans/bagit/examples/serialization/CreateZipBagExample.java index 3cccdbbff..0980859ba 100644 --- a/src/test/java/gov/loc/repository/bagit/examples/serialization/CreateZipBagExample.java +++ b/src/test/java/nl/knaw/dans/bagit/examples/serialization/CreateZipBagExample.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.examples.serialization; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.examples.serialization; import java.io.File; import java.io.IOException; @@ -13,16 +28,15 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import nl.knaw.dans.bagit.TempFolderTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.TempFolderTest; - /** * Example(s) for creating a zipped bag. */ -public class CreateZipBagExample extends TempFolderTest{ +public class CreateZipBagExample extends TempFolderTest { private Path bagRoot; private Path zippedBagPath; diff --git a/src/test/java/gov/loc/repository/bagit/hash/HasherTest.java b/src/test/java/nl/knaw/dans/bagit/hash/HasherTest.java similarity index 65% rename from src/test/java/gov/loc/repository/bagit/hash/HasherTest.java rename to src/test/java/nl/knaw/dans/bagit/hash/HasherTest.java index 96f03b1ab..c74720a28 100644 --- a/src/test/java/gov/loc/repository/bagit/hash/HasherTest.java +++ b/src/test/java/nl/knaw/dans/bagit/hash/HasherTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.hash; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.hash; import java.io.File; import java.io.IOException; @@ -12,7 +27,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.PrivateConstructorTest; public class HasherTest extends PrivateConstructorTest { diff --git a/src/test/java/nl/knaw/dans/bagit/hash/HasherUrlTest.java b/src/test/java/nl/knaw/dans/bagit/hash/HasherUrlTest.java new file mode 100644 index 000000000..8b179a3ca --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/hash/HasherUrlTest.java @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.hash; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.ProtocolException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.atomic.AtomicInteger; + +public class HasherUrlTest { + + private HttpServer server; + private static final String TEST_DATA = "This is some test data that should be hashed correctly even with retries and range requests."; + private static final byte[] TEST_DATA_BYTES = TEST_DATA.getBytes(StandardCharsets.UTF_8); + private AtomicInteger requestCount = new AtomicInteger(0); + + @BeforeEach + public void setup() throws IOException { + server = HttpServer.create(new InetSocketAddress(0), 0); + server.createContext("/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + requestCount.incrementAndGet(); + String range = exchange.getRequestHeaders().getFirst("Range"); + + if (range == null) { + // Full request + exchange.getResponseHeaders().set("Accept-Ranges", "bytes"); + exchange.getResponseHeaders().set("Content-Length", String.valueOf(TEST_DATA_BYTES.length)); + exchange.sendResponseHeaders(200, TEST_DATA_BYTES.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES); + } + } else if (range.startsWith("bytes=")) { + // Range request + String[] parts = range.substring(6).split("-"); + int start = Integer.parseInt(parts[0]); + int end = parts.length > 1 && !parts[1].isEmpty() ? Integer.parseInt(parts[1]) : TEST_DATA_BYTES.length - 1; + + if (start >= TEST_DATA_BYTES.length) { + exchange.sendResponseHeaders(416, -1); + return; + } + + int length = end - start + 1; + exchange.getResponseHeaders().set("Content-Range", "bytes " + start + "-" + end + "/" + TEST_DATA_BYTES.length); + exchange.sendResponseHeaders(206, length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES, start, length); + } + } else { + exchange.sendResponseHeaders(400, -1); + } + } + }); + server.start(); + } + + @AfterEach + public void teardown() { + if (server != null) { + server.stop(0); + } + System.clearProperty("nl.knaw.dans.bagit.hash.chunkSize"); + System.clearProperty("nl.knaw.dans.bagit.hash.maxRetries"); + System.clearProperty("nl.knaw.dans.bagit.hash.retrySleepMs"); + System.clearProperty("nl.knaw.dans.bagit.hash.fallBackToFullStreamOnRangeFail"); + } + + @Test + public void testHashWithRangeRequests() throws IOException, NoSuchAlgorithmException { + System.setProperty("nl.knaw.dans.bagit.hash.chunkSize", "10"); + URL url = new URL("http://localhost:" + server.getAddress().getPort() + "/test"); + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + String hash = Hasher.hash(url, md); + + // Expected SHA-1 of TEST_DATA + MessageDigest expectedMd = MessageDigest.getInstance("SHA-1"); + expectedMd.update(TEST_DATA_BYTES); + byte[] digest = expectedMd.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + String expectedHash = sb.toString(); + + Assertions.assertEquals(expectedHash, hash); + // With chunk size 10 and total length ~90, we expect around 10-11 requests (1 HEAD/initial + chunks) + Assertions.assertTrue(requestCount.get() > 1); + } + + @Test + public void testHashWithRetries() throws IOException, NoSuchAlgorithmException { + server.removeContext("/test"); + AtomicInteger failCount = new AtomicInteger(0); + server.createContext("/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + int count = requestCount.incrementAndGet(); + if (count == 2 || count == 3) { + // Fail the second and third request (first chunk requests) + exchange.sendResponseHeaders(500, -1); + return; + } + + String range = exchange.getRequestHeaders().getFirst("Range"); + if (range == null) { + exchange.getResponseHeaders().set("Accept-Ranges", "bytes"); + exchange.getResponseHeaders().set("Content-Length", String.valueOf(TEST_DATA_BYTES.length)); + exchange.sendResponseHeaders(200, TEST_DATA_BYTES.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES); + } + } else { + String[] parts = range.substring(6).split("-"); + int start = Integer.parseInt(parts[0]); + int end = parts.length > 1 && !parts[1].isEmpty() ? Integer.parseInt(parts[1]) : TEST_DATA_BYTES.length - 1; + int length = end - start + 1; + exchange.getResponseHeaders().set("Content-Range", "bytes " + start + "-" + end + "/" + TEST_DATA_BYTES.length); + exchange.sendResponseHeaders(206, length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES, start, length); + } + } + } + }); + + System.setProperty("nl.knaw.dans.bagit.hash.chunkSize", "20"); + System.setProperty("nl.knaw.dans.bagit.hash.maxRetries", "3"); + System.setProperty("nl.knaw.dans.bagit.hash.retrySleepMs", "100"); + + URL url = new URL("http://localhost:" + server.getAddress().getPort() + "/test"); + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + String hash = Hasher.hash(url, md); + + MessageDigest expectedMd = MessageDigest.getInstance("SHA-1"); + expectedMd.update(TEST_DATA_BYTES); + byte[] digest = expectedMd.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + String expectedHash = sb.toString(); + + Assertions.assertEquals(expectedHash, hash); + Assertions.assertTrue(requestCount.get() >= 5); // 1 initial + 2 failed + successful chunks + } + + @Test + public void testHashWithoutRangeSupport() throws IOException, NoSuchAlgorithmException { + server.removeContext("/test"); + server.createContext("/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + requestCount.incrementAndGet(); + // No Accept-Ranges header + exchange.getResponseHeaders().set("Content-Length", String.valueOf(TEST_DATA_BYTES.length)); + exchange.sendResponseHeaders(200, TEST_DATA_BYTES.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES); + } + } + }); + + URL url = new URL("http://localhost:" + server.getAddress().getPort() + "/test"); + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + String hash = Hasher.hash(url, md); + + MessageDigest expectedMd = MessageDigest.getInstance("SHA-1"); + expectedMd.update(TEST_DATA_BYTES); + byte[] digest = expectedMd.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + String expectedHash = sb.toString(); + + Assertions.assertEquals(expectedHash, hash); + // Only 1 request: the first range request which returns 200 OK and is treated as the full stream + Assertions.assertEquals(1, requestCount.get()); + } + + @Test + public void testHashThrowsProtocolExceptionWhenLaterChunkReturnsOk() throws IOException, NoSuchAlgorithmException { + server.removeContext("/test"); + server.createContext("/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + int count = requestCount.incrementAndGet(); + String range = exchange.getRequestHeaders().getFirst("Range"); + if (count == 2) { + exchange.sendResponseHeaders(200, TEST_DATA_BYTES.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES); + } + return; + } + + String[] parts = range.substring(6).split("-"); + int start = Integer.parseInt(parts[0]); + int end = parts.length > 1 && !parts[1].isEmpty() ? Integer.parseInt(parts[1]) : TEST_DATA_BYTES.length - 1; + int actualEnd = Math.min(end, TEST_DATA_BYTES.length - 1); + int length = actualEnd - start + 1; + exchange.getResponseHeaders().set("Content-Range", "bytes " + start + "-" + actualEnd + "/" + TEST_DATA_BYTES.length); + exchange.sendResponseHeaders(206, length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES, start, length); + } + } + }); + + System.setProperty("nl.knaw.dans.bagit.hash.chunkSize", "10"); + System.setProperty("nl.knaw.dans.bagit.hash.maxRetries", "3"); + + URL url = new URL("http://localhost:" + server.getAddress().getPort() + "/test"); + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + Assertions.assertThrows(ProtocolException.class, () -> Hasher.hash(url, md)); + Assertions.assertEquals(2, requestCount.get()); + } + + @Test + public void testHashWithFailedFirstRangeRequest() throws IOException, NoSuchAlgorithmException { + server.removeContext("/test"); + server.createContext("/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + requestCount.incrementAndGet(); + if (exchange.getRequestHeaders().containsKey("Range")) { + // Fail range request after 5 attempts (to trigger fallback) + exchange.sendResponseHeaders(405, -1); + } else { + exchange.getResponseHeaders().set("Content-Length", String.valueOf(TEST_DATA_BYTES.length)); + exchange.sendResponseHeaders(200, TEST_DATA_BYTES.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(TEST_DATA_BYTES); + } + } + } + }); + + System.setProperty("nl.knaw.dans.bagit.hash.maxRetries", "1"); + System.setProperty("nl.knaw.dans.bagit.hash.fallBackToFullStreamOnRangeFail", "true"); + try { + URL url = new URL("http://localhost:" + server.getAddress().getPort() + "/test"); + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + String hash = Hasher.hash(url, md); + + MessageDigest expectedMd = MessageDigest.getInstance("SHA-1"); + expectedMd.update(TEST_DATA_BYTES); + byte[] digest = expectedMd.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + String expectedHash = sb.toString(); + + Assertions.assertEquals(expectedHash, hash); + // Expect 2 requests: 1 range request (failed) and 1 GET (fallback to full stream) + Assertions.assertEquals(2, requestCount.get()); + } finally { + System.clearProperty("nl.knaw.dans.bagit.hash.maxRetries"); + System.clearProperty("nl.knaw.dans.bagit.hash.fallBackToFullStreamOnRangeFail"); + } + } + + @Test + public void testHashWithFileUrl() throws IOException, NoSuchAlgorithmException { + java.nio.file.Path tempFile = java.nio.file.Files.createTempFile("hasher-file-url-test", ".txt"); + try { + java.nio.file.Files.write(tempFile, TEST_DATA_BYTES); + URL fileUrl = tempFile.toUri().toURL(); + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + String hash = Hasher.hash(fileUrl, md); + + MessageDigest expectedMd = MessageDigest.getInstance("SHA-1"); + expectedMd.update(TEST_DATA_BYTES); + byte[] digest = expectedMd.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + String expectedHash = sb.toString(); + + Assertions.assertEquals(expectedHash, hash); + } finally { + java.nio.file.Files.deleteIfExists(tempFile); + } + } +} diff --git a/src/test/java/gov/loc/repository/bagit/reader/BagReaderTest.java b/src/test/java/nl/knaw/dans/bagit/reader/BagReaderTest.java similarity index 89% rename from src/test/java/gov/loc/repository/bagit/reader/BagReaderTest.java rename to src/test/java/nl/knaw/dans/bagit/reader/BagReaderTest.java index b742a57e1..1e40e078e 100644 --- a/src/test/java/gov/loc/repository/bagit/reader/BagReaderTest.java +++ b/src/test/java/nl/knaw/dans/bagit/reader/BagReaderTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.File; import java.net.URL; @@ -10,15 +25,15 @@ import java.util.ArrayList; import java.util.List; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.FetchItem; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.FetchItem; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.domain.Metadata; -import gov.loc.repository.bagit.domain.Version; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.domain.Metadata; +import nl.knaw.dans.bagit.domain.Version; public class BagReaderTest { private BagReader sut; diff --git a/src/test/java/gov/loc/repository/bagit/reader/BagitTextFileReaderTest.java b/src/test/java/nl/knaw/dans/bagit/reader/BagitTextFileReaderTest.java similarity index 79% rename from src/test/java/gov/loc/repository/bagit/reader/BagitTextFileReaderTest.java rename to src/test/java/nl/knaw/dans/bagit/reader/BagitTextFileReaderTest.java index 5353f667e..5d576ee23 100644 --- a/src/test/java/gov/loc/repository/bagit/reader/BagitTextFileReaderTest.java +++ b/src/test/java/nl/knaw/dans/bagit/reader/BagitTextFileReaderTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.io.File; import java.lang.reflect.InvocationTargetException; @@ -10,13 +25,13 @@ import java.util.Arrays; import java.util.List; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.UnparsableVersionException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.UnparsableVersionException; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.Version; public class BagitTextFileReaderTest extends PrivateConstructorTest { @@ -35,7 +50,7 @@ public void testLinesMatchesStrict() throws Exception{ public void testFirstLineMatchesStrict() throws Exception{ //should fail because it has spaces before the colon List lines = Arrays.asList("BagIt-Version : 1.0", "Tag-File-Character-Encoding: UTF-8"); - Assertions.assertThrows(InvalidBagitFileFormatException.class, + Assertions.assertThrows(InvalidBagitFileFormatException.class, () -> { BagitTextFileReader.throwErrorIfLinesDoNotMatchStrict(lines); }); } @@ -56,7 +71,7 @@ public void testMatchesStrictWithTooManyLines() throws Exception{ } @Test - public void testParseVersionWithBadVersion() throws UnparsableVersionException{ + public void testParseVersionWithBadVersion() throws UnparsableVersionException { Assertions.assertThrows(UnparsableVersionException.class, () -> { BagitTextFileReader.parseVersion("someVersionThatIsUnparsable"); }); } diff --git a/src/test/java/gov/loc/repository/bagit/reader/FetchReaderTest.java b/src/test/java/nl/knaw/dans/bagit/reader/FetchReaderTest.java similarity index 81% rename from src/test/java/gov/loc/repository/bagit/reader/FetchReaderTest.java rename to src/test/java/nl/knaw/dans/bagit/reader/FetchReaderTest.java index 0bef9a31f..fbff0bba6 100644 --- a/src/test/java/gov/loc/repository/bagit/reader/FetchReaderTest.java +++ b/src/test/java/nl/knaw/dans/bagit/reader/FetchReaderTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.lang.reflect.InvocationTargetException; import java.net.MalformedURLException; @@ -9,16 +24,16 @@ import java.util.Arrays; import java.util.List; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.FetchItem; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; +import nl.knaw.dans.bagit.PrivateConstructorTest; public class FetchReaderTest extends PrivateConstructorTest { @@ -71,7 +86,7 @@ public void testReadFetchWithSizeSpecified() throws Exception{ @Test public void testReadBlankLinesThrowsException() throws Exception{ Path fetchFile = Paths.get(getClass().getClassLoader().getResource("fetchFiles/fetchWithBlankLines.txt").toURI()); - Assertions.assertThrows(InvalidBagitFileFormatException.class, + Assertions.assertThrows(InvalidBagitFileFormatException.class, () -> { FetchReader.readFetch(fetchFile, StandardCharsets.UTF_8, Paths.get("/foo")); }); } @@ -85,7 +100,7 @@ public void testReadWindowsSpecialDirMaliciousFetchThrowsException() throws Exce @Test public void testReadUpADirMaliciousFetchThrowsException() throws Exception{ Path fetchFile = Paths.get(getClass().getClassLoader().getResource("maliciousFetchFile/upAdirectoryReference.txt").toURI()); - Assertions.assertThrows(MaliciousPathException.class, + Assertions.assertThrows(MaliciousPathException.class, () -> { FetchReader.readFetch(fetchFile, StandardCharsets.UTF_8, Paths.get("/bar")); }); } @@ -103,16 +118,4 @@ public void testReadFileUrlMaliciousFetchThrowsException() throws Exception{ Assertions.assertThrows(MaliciousPathException.class, () -> { FetchReader.readFetch(fetchFile, StandardCharsets.UTF_8, Paths.get("/bar")); }); } - - @Test - public void foo(){ - String regex = ".*[ \t]*(\\d*|-)[ \t]*.*"; - String test1 = "http://localhost/foo/data/test2.txt - ~/foo/bar/ham.txt"; - String test2 = "http://localhost/foo/data/dir1/test3.txt 100057 data/dir1/test3.txt"; - String test3 = "http://localhost/foo/data/dir1/test3.txt \t 100057 \t data/dir1/test3.txt"; - - System.err.println(test1.matches(regex)); - System.err.println(test2.matches(regex)); - System.err.println(test3.matches(regex)); - } } diff --git a/src/test/java/gov/loc/repository/bagit/reader/KeyValueReaderTest.java b/src/test/java/nl/knaw/dans/bagit/reader/KeyValueReaderTest.java similarity index 61% rename from src/test/java/gov/loc/repository/bagit/reader/KeyValueReaderTest.java rename to src/test/java/nl/knaw/dans/bagit/reader/KeyValueReaderTest.java index db34aee33..a82e0dc65 100644 --- a/src/test/java/gov/loc/repository/bagit/reader/KeyValueReaderTest.java +++ b/src/test/java/nl/knaw/dans/bagit/reader/KeyValueReaderTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; @@ -8,8 +23,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.exceptions.InvalidBagMetadataException; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.exceptions.InvalidBagMetadataException; public class KeyValueReaderTest extends PrivateConstructorTest { diff --git a/src/test/java/gov/loc/repository/bagit/reader/ManifestReaderTest.java b/src/test/java/nl/knaw/dans/bagit/reader/ManifestReaderTest.java similarity index 71% rename from src/test/java/gov/loc/repository/bagit/reader/ManifestReaderTest.java rename to src/test/java/nl/knaw/dans/bagit/reader/ManifestReaderTest.java index 7e32cba03..222bba0c7 100644 --- a/src/test/java/gov/loc/repository/bagit/reader/ManifestReaderTest.java +++ b/src/test/java/nl/knaw/dans/bagit/reader/ManifestReaderTest.java @@ -1,20 +1,35 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; public class ManifestReaderTest extends PrivateConstructorTest { @@ -36,7 +51,7 @@ public void testReadAllManifests() throws Exception{ @Test public void testReadUpDirectoryMaliciousManifestThrowsException() throws Exception{ Path manifestFile = Paths.get(getClass().getClassLoader().getResource("maliciousManifestFile/upAdirectoryReference.txt").toURI()); - Assertions.assertThrows(MaliciousPathException.class, + Assertions.assertThrows(MaliciousPathException.class, () -> { ManifestReader.readChecksumFileMap(manifestFile, Paths.get("/foo"), StandardCharsets.UTF_8); }); } @@ -58,7 +73,7 @@ public void testReadFileUrlMaliciousManifestThrowsException() throws Exception{ @Test public void testReadWindowsSpecialDirMaliciousManifestThrowsException() throws Exception{ Path manifestFile = Paths.get(getClass().getClassLoader().getResource("maliciousManifestFile/windowsSpecialDirectoryName.txt").toURI()); - Assertions.assertThrows(InvalidBagitFileFormatException.class, + Assertions.assertThrows(InvalidBagitFileFormatException.class, () -> { ManifestReader.readChecksumFileMap(manifestFile, Paths.get("/foo"), StandardCharsets.UTF_8); }); } } diff --git a/src/test/java/gov/loc/repository/bagit/reader/MetadataReaderTest.java b/src/test/java/nl/knaw/dans/bagit/reader/MetadataReaderTest.java similarity index 75% rename from src/test/java/gov/loc/repository/bagit/reader/MetadataReaderTest.java rename to src/test/java/nl/knaw/dans/bagit/reader/MetadataReaderTest.java index 6f95f7921..2e308135a 100644 --- a/src/test/java/gov/loc/repository/bagit/reader/MetadataReaderTest.java +++ b/src/test/java/nl/knaw/dans/bagit/reader/MetadataReaderTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; @@ -11,7 +26,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.PrivateConstructorTest; public class MetadataReaderTest extends PrivateConstructorTest { @@ -29,7 +44,7 @@ public void testReadBagMetadata() throws Exception{ expectedValues.add(new SimpleImmutableEntry<>("Contact-Phone", "+1 408-555-1212")); expectedValues.add(new SimpleImmutableEntry<>("Contact-Email", "ej@spengler.edu")); expectedValues.add(new SimpleImmutableEntry<>("External-Description", "Uncompressed greyscale TIFF images from the" + System.lineSeparator() + - " Yoshimuri papers collection.")); + "Yoshimuri papers collection.")); expectedValues.add(new SimpleImmutableEntry<>("Bagging-Date", "2008-01-15")); expectedValues.add(new SimpleImmutableEntry<>("External-Identifier", "spengler_yoshimuri_001")); expectedValues.add(new SimpleImmutableEntry<>("Bag-Size", "260 GB")); @@ -37,7 +52,7 @@ public void testReadBagMetadata() throws Exception{ expectedValues.add(new SimpleImmutableEntry<>("Bag-Count", "1 of 15")); expectedValues.add(new SimpleImmutableEntry<>("Internal-Sender-Identifier", "/storage/images/yoshimuri")); expectedValues.add(new SimpleImmutableEntry<>("Internal-Sender-Description", "Uncompressed greyscale TIFFs created from" + System.lineSeparator() + - " microfilm.")); + "microfilm.")); expectedValues.add(new SimpleImmutableEntry<>("Bag-Count", "1 of 15")); //test duplicate Path bagInfoFile = Paths.get(getClass().getClassLoader().getResource("baginfoFiles").toURI()); diff --git a/src/test/java/gov/loc/repository/bagit/reader/TagFileReaderTest.java b/src/test/java/nl/knaw/dans/bagit/reader/TagFileReaderTest.java similarity index 73% rename from src/test/java/gov/loc/repository/bagit/reader/TagFileReaderTest.java rename to src/test/java/nl/knaw/dans/bagit/reader/TagFileReaderTest.java index a2d643a28..90e065d0d 100644 --- a/src/test/java/gov/loc/repository/bagit/reader/TagFileReaderTest.java +++ b/src/test/java/nl/knaw/dans/bagit/reader/TagFileReaderTest.java @@ -1,14 +1,28 @@ -package gov.loc.repository.bagit.reader; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.reader; import java.nio.file.Path; import java.nio.file.Paths; +import nl.knaw.dans.bagit.exceptions.InvalidBagitFileFormatException; +import nl.knaw.dans.bagit.exceptions.MaliciousPathException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; - public class TagFileReaderTest { @Test @@ -36,14 +50,14 @@ public void testCreateFileFromManifestWithURISyntax() throws Exception{ @Test public void testBackslashThrowsException() throws Exception{ Path bagRootDir = Paths.get("foo"); - Assertions.assertThrows(InvalidBagitFileFormatException.class, + Assertions.assertThrows(InvalidBagitFileFormatException.class, () -> { TagFileReader.createFileFromManifest(bagRootDir, "data\\bar\\ham.txt"); }); } @Test public void testOutsideDataDirReferenceThrowsException() throws Exception{ Path bagRootDir = Paths.get("foo"); - Assertions.assertThrows(MaliciousPathException.class, + Assertions.assertThrows(MaliciousPathException.class, () -> { TagFileReader.createFileFromManifest(bagRootDir, "/bar/ham.txt"); }); } diff --git a/src/test/java/gov/loc/repository/bagit/util/PathUtilsTest.java b/src/test/java/nl/knaw/dans/bagit/util/PathUtilsTest.java similarity index 83% rename from src/test/java/gov/loc/repository/bagit/util/PathUtilsTest.java rename to src/test/java/nl/knaw/dans/bagit/util/PathUtilsTest.java index 243536f36..571cf9489 100644 --- a/src/test/java/gov/loc/repository/bagit/util/PathUtilsTest.java +++ b/src/test/java/nl/knaw/dans/bagit/util/PathUtilsTest.java @@ -1,17 +1,32 @@ -package gov.loc.repository.bagit.util; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.util; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; +import nl.knaw.dans.bagit.domain.Bag; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.TestUtils; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Version; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.TestUtils; +import nl.knaw.dans.bagit.domain.Version; public class PathUtilsTest extends PrivateConstructorTest { diff --git a/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierRedirectTest.java b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierRedirectTest.java new file mode 100644 index 000000000..f7f5d7297 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierRedirectTest.java @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.reader.BagReader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +public class BagVerifierRedirectTest extends TempFolderTest { + private HttpServer server1; + private HttpServer server2; + private BagVerifier sut; + private BagReader reader; + private int port1; + private int port2; + + @BeforeEach + public void setup() throws IOException { + sut = new BagVerifier(); + reader = new BagReader(); + + server1 = HttpServer.create(new InetSocketAddress(0), 0); + server1.setExecutor(null); + server1.start(); + port1 = server1.getAddress().getPort(); + + server2 = HttpServer.create(new InetSocketAddress(0), 0); + server2.setExecutor(null); + server2.start(); + port2 = server2.getAddress().getPort(); + } + + @AfterEach + public void tearDown() { + if (server1 != null) server1.stop(0); + if (server2 != null) server2.stop(0); + if (sut != null) sut.close(); + } + + @Test + public void testRedirectStripsHeadersOnHostChange() throws Exception { + final String content = "final content"; + final String authHeaderName = "X-Dataverse-key"; + final String authHeaderValue = "secret"; + final AtomicBoolean headersSentToServer2 = new AtomicBoolean(false); + + // Server 1 redirects to Server 2 + server1.createContext("/api/access/datafile/1", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if (authHeaderValue.equals(exchange.getRequestHeaders().getFirst(authHeaderName))) { + exchange.getResponseHeaders().set("Location", "http://localhost:" + port2 + "/storage/file1?token=xyz"); + exchange.sendResponseHeaders(302, -1); + } else { + exchange.sendResponseHeaders(401, -1); + } + } + }); + + // Server 2 returns content but fails if auth header is present + server2.createContext("/storage/file1", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if (exchange.getRequestHeaders().containsKey(authHeaderName)) { + headersSentToServer2.set(true); + // Simulate 403 Forbidden because S3 rejects requests with unknown/extra auth headers + exchange.sendResponseHeaders(403, -1); + } else { + byte[] response = content.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + } + }); + + Path bagDir = Files.createTempDirectory(folder, "redirect-headers-bag"); + Files.write(bagDir.resolve("bagit.txt"), "BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n".getBytes()); + Files.createDirectory(bagDir.resolve("data")); + + MessageDigest md5 = MessageDigest.getInstance("MD5"); + md5.update(content.getBytes(StandardCharsets.UTF_8)); + String hash = formatMessageDigest(md5.digest()); + + Files.write(bagDir.resolve("manifest-md5.txt"), (hash + " data/test.txt\n").getBytes()); + + URL remoteUrl = new URL("http://localhost:" + port1 + "/api/access/datafile/1"); + Files.write(bagDir.resolve("fetch.txt"), (remoteUrl.toString() + " - data/test.txt\n").getBytes()); + + Bag bag = reader.read(bagDir); + + Map headers = new HashMap<>(); + headers.put(authHeaderName, authHeaderValue); + + // This should now PASS because Hasher.java strips headers on host change + sut.isValid(bag, true, true, headers); + + Assertions.assertFalse(headersSentToServer2.get(), "Headers should NOT have been sent to server2"); + } + + private String formatMessageDigest(final byte[] digest) { + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } +} diff --git a/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierRemoteTest.java b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierRemoteTest.java new file mode 100644 index 000000000..6096097c1 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierRemoteTest.java @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.reader.BagReader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class BagVerifierRemoteTest extends TempFolderTest { + private HttpServer server; + private BagVerifier sut; + private BagReader reader; + private int port; + + @BeforeEach + public void setup() throws IOException { + sut = new BagVerifier(); + reader = new BagReader(); + server = HttpServer.create(new InetSocketAddress(0), 0); + server.setExecutor(null); + server.start(); + port = server.getAddress().getPort(); + } + + @AfterEach + public void tearDown() { + if (server != null) server.stop(0); + if (sut != null) sut.close(); + } + + @Test + public void testAuthenticatedRangeRequest() throws Exception { + final String content = "this is some test content for range requests"; + final String authHeaderName = "X-Dataverse-key"; + final String authHeaderValue = "secret-key"; + final AtomicInteger rangeRequestCount = new AtomicInteger(0); + + server.createContext("/datafile", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if (!authHeaderValue.equals(exchange.getRequestHeaders().getFirst(authHeaderName))) { + exchange.sendResponseHeaders(401, -1); + return; + } + + String range = exchange.getRequestHeaders().getFirst("Range"); + if (range != null && range.startsWith("bytes=")) { + rangeRequestCount.incrementAndGet(); + String[] parts = range.substring(6).split("-"); + int start = Integer.parseInt(parts[0]); + int end = Integer.parseInt(parts[1]); + byte[] fullContent = content.getBytes(StandardCharsets.UTF_8); + int actualEnd = Math.min(end, fullContent.length - 1); + int length = actualEnd - start + 1; + + exchange.getResponseHeaders().set("Content-Range", "bytes " + start + "-" + actualEnd + "/" + fullContent.length); + exchange.sendResponseHeaders(206, length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(fullContent, start, length); + } + } else { + byte[] response = content.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + } + }); + + Path bagDir = Files.createTempDirectory(folder, "remote-bag"); + Files.write(bagDir.resolve("bagit.txt"), "BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n".getBytes()); + Files.createDirectory(bagDir.resolve("data")); + + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + sha1.update(content.getBytes(StandardCharsets.UTF_8)); + String hash = formatMessageDigest(sha1.digest()); + + Files.write(bagDir.resolve("manifest-sha1.txt"), (hash + " data/test.txt\n").getBytes()); + + URL remoteUrl = new URL("http://localhost:" + port + "/datafile"); + Files.write(bagDir.resolve("fetch.txt"), (remoteUrl.toString() + " " + content.length() + " data/test.txt\n").getBytes()); + + Bag bag = reader.read(bagDir); + + Map headers = new HashMap<>(); + headers.put(authHeaderName, authHeaderValue); + + // Use a small chunk size to force multiple range requests + sut.setChunkSize(10); + sut.isValid(bag, true, true, headers); + + Assertions.assertTrue(rangeRequestCount.get() > 1, "Should have used multiple range requests, but used: " + rangeRequestCount.get()); + } + + @Test + public void testFallbackToFullDownloadWhenRangeNotSupported() throws Exception { + final String content = "this content will be downloaded in one go"; + final AtomicBoolean rangeAttempted = new AtomicBoolean(false); + + server.createContext("/datafile", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if (exchange.getRequestHeaders().containsKey("Range")) { + rangeAttempted.set(true); + // Server ignores range and returns 200 OK + byte[] response = content.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } else { + byte[] response = content.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + } + }); + + Path bagDir = Files.createTempDirectory(folder, "remote-bag-fallback"); + Files.write(bagDir.resolve("bagit.txt"), "BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n".getBytes()); + Files.createDirectory(bagDir.resolve("data")); + + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + sha1.update(content.getBytes(StandardCharsets.UTF_8)); + String hash = formatMessageDigest(sha1.digest()); + + Files.write(bagDir.resolve("manifest-sha1.txt"), (hash + " data/test.txt\n").getBytes()); + + URL remoteUrl = new URL("http://localhost:" + port + "/datafile"); + Files.write(bagDir.resolve("fetch.txt"), (remoteUrl.toString() + " " + content.length() + " data/test.txt\n").getBytes()); + + Bag bag = reader.read(bagDir); + + sut.isValid(bag, true, true, null); + + Assertions.assertTrue(rangeAttempted.get(), "Should have attempted a range request"); + } + + private String formatMessageDigest(final byte[] digest) { + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } +} diff --git a/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierTest.java b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierTest.java new file mode 100644 index 000000000..e43e1da69 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierTest.java @@ -0,0 +1,351 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.TestUtils; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.exceptions.CorruptChecksumException; +import nl.knaw.dans.bagit.exceptions.FileNotInManifestException; +import nl.knaw.dans.bagit.exceptions.UnsupportedAlgorithmException; +import nl.knaw.dans.bagit.exceptions.VerificationException; +import nl.knaw.dans.bagit.hash.Hasher; +import nl.knaw.dans.bagit.hash.Hasher.HashOptions; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; +import nl.knaw.dans.bagit.hash.SupportedAlgorithm; +import nl.knaw.dans.bagit.reader.BagReader; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.Security; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import nl.knaw.dans.bagit.domain.FetchItem; + +public class BagVerifierTest extends TempFolderTest { + static { + if (Security.getProvider("BC") == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + private Path rootDir = Paths.get(new File("src/test/resources/bags/v0_97/bag").toURI()); + + private final BagVerifier sut = new BagVerifier(); + private final BagReader reader = new BagReader(); + + @Test + public void testValidWhenHiddenFolderNotIncluded() throws Exception { + Path copyDir = copyBagToTempFolder(rootDir); + Files.createDirectory(copyDir.resolve("data").resolve(".someHiddenFolder")); + TestUtils.makeFilesHiddenOnWindows(copyDir); + + Bag bag = reader.read(copyDir); + sut.isValid(bag, true); + } + + @Test + public void testValidWithHiddenFile() throws Exception { + Path copyDir = copyBagToTempFolder(rootDir); + Files.createFile(copyDir.resolve("data").resolve(".someHiddenFile")); + TestUtils.makeFilesHiddenOnWindows(copyDir); + + Bag bag = reader.read(copyDir); + sut.isValid(bag, true); + } + + @Test + public void testInvalidWithHiddenFile() throws Exception { + Path copyDir = copyBagToTempFolder(rootDir); + Files.createFile(copyDir.resolve("data").resolve(".someHiddenFile")); + TestUtils.makeFilesHiddenOnWindows(copyDir); + + Bag bag = reader.read(copyDir); + Assertions.assertThrows(FileNotInManifestException.class, () -> { + sut.isValid(bag, false); + }); + } + + @Test + public void testStandardSupportedAlgorithms() throws Exception { + List algorithms = Arrays.asList("md5", "sha1", "sha256", "sha512"); + for (String alg : algorithms) { + StandardSupportedAlgorithms algorithm = StandardSupportedAlgorithms.valueOf(alg.toUpperCase()); + Manifest manifest = new Manifest(algorithm); + sut.checkHashes(manifest); + } + } + + @Test + public void hash_options_on_bag_verifier_override_system_properties() { + System.setProperty(Hasher.CHUNK_SIZE_PROP, "10"); + System.setProperty(Hasher.MAX_RETRIES_PROP, "2"); + System.setProperty(Hasher.RETRY_SLEEP_MS_PROP, "30"); + try (BagVerifier sut = new BagVerifier()) { + HashOptions systemHashOptions = sut.getHashOptions(); + + Assertions.assertEquals(10, systemHashOptions.getChunkSize()); + Assertions.assertEquals(2, systemHashOptions.getMaxRetries()); + Assertions.assertEquals(30, systemHashOptions.getRetrySleepMs()); + + sut.setChunkSize(20); + sut.setMaxRetries(3); + sut.setRetrySleepMs(40); + + HashOptions hashOptions = sut.getHashOptions(); + + Assertions.assertEquals(20, hashOptions.getChunkSize()); + Assertions.assertEquals(3, hashOptions.getMaxRetries()); + Assertions.assertEquals(40, hashOptions.getRetrySleepMs()); + } + finally { + System.clearProperty(Hasher.CHUNK_SIZE_PROP); + System.clearProperty(Hasher.MAX_RETRIES_PROP); + System.clearProperty(Hasher.RETRY_SLEEP_MS_PROP); + } + } + + @Test + public void testMD5Bag() throws Exception { + Path bagDir = Paths.get("src", "test", "resources", "md5Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA1Bag() throws Exception { + Path bagDir = Paths.get("src", "test", "resources", "sha1Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA224Bag() throws Exception { + Path bagDir = Paths.get("src", "test", "resources", "sha224Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA256Bag() throws Exception { + Path bagDir = Paths.get("src", "test", "resources", "sha256Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA512Bag() throws Exception { + Path bagDir = Paths.get("src", "test", "resources", "sha512Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testVersion0_97IsValid() throws Exception { + Bag bag = reader.read(rootDir); + + sut.isValid(bag, true); + } + + @Test + public void testVersion2_0IsValid() throws Exception { + rootDir = Paths.get(new File("src/test/resources/bags/v2_0/bag").toURI()); + Bag bag = reader.read(rootDir); + + sut.isValid(bag, true); + } + + @Test + public void testIsComplete() throws Exception { + Bag bag = reader.read(rootDir); + + sut.isComplete(bag, true); + } + + @Test + public void testCorruptPayloadFile() throws Exception { + rootDir = Paths.get(new File("src/test/resources/corruptPayloadFile").toURI()); + Bag bag = reader.read(rootDir); + + Assertions.assertThrows(CorruptChecksumException.class, () -> { + sut.isValid(bag, true); + }); + } + + @Test + public void testCorruptTagFile() throws Exception { + rootDir = Paths.get(new File("src/test/resources/corruptTagFile").toURI()); + Bag bag = reader.read(rootDir); + + Assertions.assertThrows(CorruptChecksumException.class, () -> { + sut.isValid(bag, true); + }); + } + + @Test + public void testErrorWhenUnspportedAlgorithmException() throws Exception { + Path sha3BagDir = Paths.get(getClass().getClassLoader().getResource("sha3Bag").toURI()); + MySupportedNameToAlgorithmMapping mapping = new MySupportedNameToAlgorithmMapping(); + BagReader extendedReader = new BagReader(mapping); + Bag bag = extendedReader.read(sha3BagDir); + + Assertions.assertThrows(UnsupportedAlgorithmException.class, () -> { + sut.isValid(bag, true); + }); + } + + @Test + public void testVerificationExceptionIsThrownForNoSuchAlgorithmException() throws Exception { + Path unreadableFile = createFile("newFile"); + + Manifest manifest = new Manifest(new SupportedAlgorithm() { + + @Override + public String getMessageDigestName() { + return "FOO"; + } + + @Override + public String getBagitName() { + return "foo"; + } + }); + manifest.getFileToChecksumMap().put(unreadableFile, "foo"); + + Assertions.assertThrows(VerificationException.class, () -> { + sut.checkHashes(manifest); + }); + } + + @Test + public void testAddSHA3SupportViaExtension() throws Exception { + Path sha3BagDir = Paths.get(new File("src/test/resources/sha3Bag").toURI()); + MySupportedNameToAlgorithmMapping mapping = new MySupportedNameToAlgorithmMapping(); + BagReader extendedReader = new BagReader(mapping); + Bag bag = extendedReader.read(sha3BagDir); + try (BagVerifier extendedSut = new BagVerifier(mapping)) { + extendedSut.isValid(bag, true); + } + } + + /* + * Technically valid but highly discouraged + */ + @Test + public void testManifestsWithLeadingDotSlash() throws Exception { + Path bagPath = Paths.get(new File("src/test/resources/bag-with-leading-dot-slash-in-manifest").toURI()); + Bag bag = reader.read(bagPath); + + sut.isValid(bag, true); + } + + @Test + public void testCanQuickVerify() throws Exception { + Bag bag = reader.read(rootDir); + boolean canQuickVerify = BagVerifier.canQuickVerify(bag); + Assertions.assertFalse(canQuickVerify, + "Since " + bag.getRootDir() + " DOES NOT contain the metadata Payload-Oxum then it should return false!"); + + Path passingRootDir = Paths.get(new File("src/test/resources/bags/v0_94/bag").toURI()); + bag = reader.read(passingRootDir); + canQuickVerify = BagVerifier.canQuickVerify(bag); + Assertions.assertTrue(canQuickVerify, + "Since " + bag.getRootDir() + " DOES contain the metadata Payload-Oxum then it should return true!"); + } + + @Test + public void testQuickVerify() throws Exception { + Path passingRootDir = Paths.get(new File("src/test/resources/bags/v0_94/bag").toURI()); + Bag bag = reader.read(passingRootDir); + + BagVerifier.quicklyVerify(bag); + } + + @Test + public void testHoleyBag() throws Exception { + Path bagDir = Paths.get("src", "test", "resources", "md5Bag"); + Path copyDir = copyBagToTempFolder(bagDir); + Path readme = copyDir.resolve("data/readme.txt"); + byte[] content = Files.readAllBytes(readme); + Files.delete(readme); + + // Create a local file to serve as "remote" resource + Path remoteFile = copyDir.resolve("remote-readme.txt"); + Files.write(remoteFile, content); + URL remoteUrl = remoteFile.toUri().toURL(); + + // Create fetch.txt + Path fetchFile = copyDir.resolve("fetch.txt"); + // Format of fetch.txt: url length path + // IMPORTANT: BagReader uses relative paths from root for fetch items, + // but they should NOT have 'data/' prefix if they are in data directory? + // Actually, BagIt spec says it's the path relative to the bag root. + String fetchLine = remoteUrl.toString() + " " + content.length + " data/readme.txt\n"; + Files.write(fetchFile, fetchLine.getBytes()); + + Bag bag = reader.read(copyDir); + + // With the new logic, it should be valid even without explicitly passing true, + // because fetch.txt is present and contains data/readme.txt + sut.isValid(bag, true); + + // It should also be valid if we explicitly pass true + sut.isValid(bag, true, true); + } + + @Test + public void testIgnoredFetchItemsSkipped() throws Exception { + Path bagDir = Paths.get("src", "test", "resources", "md5Bag"); + Path copyDir = copyBagToTempFolder(bagDir); + Path readme = copyDir.resolve("data/readme.txt"); + byte[] content = Files.readAllBytes(readme); + Files.delete(readme); + + // Create a local file to serve as "remote" resource with CORRUPT content + Path remoteFile = copyDir.resolve("remote-readme.txt"); + Files.write(remoteFile, "corrupt content".getBytes()); + URL remoteUrl = remoteFile.toUri().toURL(); + + Path fetchFile = copyDir.resolve("fetch.txt"); + String fetchLine = remoteUrl.toString() + " " + content.length + " data/readme.txt\n"; + Files.write(fetchFile, fetchLine.getBytes()); + + Bag bag = reader.read(copyDir); + + // If we don't ignore it, it should fail due to corrupt checksum + Assertions.assertThrows(CorruptChecksumException.class, () -> { + sut.isValid(bag, true); + }); + + // Now ignore it + Map> ignored = new HashMap<>(); + ignored.put(StandardSupportedAlgorithms.MD5, Collections.singletonList(bag.getItemsToFetch().get(0))); + + // Verification should pass because the hashing is skipped + sut.isValid(bag, true, true, null, null, ignored); + } +} diff --git a/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierUrlConfigTest.java b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierUrlConfigTest.java new file mode 100644 index 000000000..06a129dd7 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/verify/BagVerifierUrlConfigTest.java @@ -0,0 +1,238 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.reader.BagReader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +public class BagVerifierUrlConfigTest extends TempFolderTest { + private HttpServer server; + private BagVerifier sut; + private BagReader reader; + private int port; + + @BeforeEach + public void setup() throws IOException { + sut = new BagVerifier(); + reader = new BagReader(); + server = HttpServer.create(new InetSocketAddress(0), 0); + server.setExecutor(null); + server.start(); + port = server.getAddress().getPort(); + } + + @AfterEach + public void tearDown() { + if (server != null) server.stop(0); + if (sut != null) sut.close(); + } + + @Test + public void testUrlSpecificHeaders() throws Exception { + final String content = "test content"; + final String globalHeaderName = "X-Global"; + final String globalHeaderValue = "global-val"; + final String urlHeaderName = "X-Url-Specific"; + final String urlHeaderValue = "url-val"; + + final AtomicBoolean globalReceived = new AtomicBoolean(false); + final AtomicBoolean urlSpecificReceived = new AtomicBoolean(false); + + server.createContext("/specific", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if (globalHeaderValue.equals(exchange.getRequestHeaders().getFirst(globalHeaderName))) { + globalReceived.set(true); + } + if (urlHeaderValue.equals(exchange.getRequestHeaders().getFirst(urlHeaderName))) { + urlSpecificReceived.set(true); + } + + byte[] response = content.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + }); + + Path bagDir = Files.createTempDirectory(folder, "url-config-bag"); + Files.write(bagDir.resolve("bagit.txt"), "BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n".getBytes()); + Files.createDirectory(bagDir.resolve("data")); + + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + sha1.update(content.getBytes(StandardCharsets.UTF_8)); + String hash = formatMessageDigest(sha1.digest()); + + Files.write(bagDir.resolve("manifest-sha1.txt"), (hash + " data/test.txt\n").getBytes()); + + URL remoteUrl = new URL("http://localhost:" + port + "/specific"); + Files.write(bagDir.resolve("fetch.txt"), (remoteUrl.toString() + " " + content.length() + " data/test.txt\n").getBytes()); + + Bag bag = reader.read(bagDir); + + Map globalHeaders = new HashMap<>(); + globalHeaders.put(globalHeaderName, globalHeaderValue); + + Map> urlConfigs = new HashMap<>(); + Map specificHeaders = new HashMap<>(); + specificHeaders.put(urlHeaderName, urlHeaderValue); + urlConfigs.put("http://localhost:" + port + "/specific", specificHeaders); + + sut.isValid(bag, true, true, globalHeaders, urlConfigs); + + Assertions.assertTrue(globalReceived.get(), "Global header should have been received"); + Assertions.assertTrue(urlSpecificReceived.get(), "URL specific header should have been received"); + } + + @Test + public void testUrlPrefixMatching() throws Exception { + final String content = "test content"; + final String headerName = "X-Url-Specific"; + final String headerValue = "url-val"; + + final AtomicBoolean headerReceived = new AtomicBoolean(false); + + server.createContext("/prefix/specific", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if (headerValue.equals(exchange.getRequestHeaders().getFirst(headerName))) { + headerReceived.set(true); + } + + byte[] response = content.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + }); + + Path bagDir = Files.createTempDirectory(folder, "url-prefix-bag"); + Files.write(bagDir.resolve("bagit.txt"), "BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n".getBytes()); + Files.createDirectory(bagDir.resolve("data")); + + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + sha1.update(content.getBytes(StandardCharsets.UTF_8)); + String hash = formatMessageDigest(sha1.digest()); + + Files.write(bagDir.resolve("manifest-sha1.txt"), (hash + " data/test.txt\n").getBytes()); + + URL remoteUrl = new URL("http://localhost:" + port + "/prefix/specific"); + Files.write(bagDir.resolve("fetch.txt"), (remoteUrl.toString() + " " + content.length() + " data/test.txt\n").getBytes()); + + Bag bag = reader.read(bagDir); + + Map> urlConfigs = new HashMap<>(); + Map specificHeaders = new HashMap<>(); + specificHeaders.put(headerName, headerValue); + // Using a prefix + urlConfigs.put("http://localhost:" + port + "/prefix", specificHeaders); + + sut.isValid(bag, true, true, null, urlConfigs); + + Assertions.assertTrue(headerReceived.get(), "URL specific header should have been received because of prefix match"); + } + + @Test + public void testMultipleUrlConfigs() throws Exception { + final String content1 = "content 1"; + final String content2 = "content 2"; + final String header1 = "X-H1"; + final String header2 = "X-H2"; + + final AtomicBoolean h1Received = new AtomicBoolean(false); + final AtomicBoolean h2Received = new AtomicBoolean(false); + + server.createContext("/url1", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if ("v1".equals(exchange.getRequestHeaders().getFirst(header1))) h1Received.set(true); + byte[] response = content1.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { os.write(response); } + } + }); + server.createContext("/url2", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + if ("v2".equals(exchange.getRequestHeaders().getFirst(header2))) h2Received.set(true); + byte[] response = content2.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { os.write(response); } + } + }); + + Path bagDir = Files.createTempDirectory(folder, "multi-url-bag"); + Files.write(bagDir.resolve("bagit.txt"), "BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n".getBytes()); + Files.createDirectory(bagDir.resolve("data")); + + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + + sha1.update(content1.getBytes(StandardCharsets.UTF_8)); + String hash1 = formatMessageDigest(sha1.digest()); + sha1.reset(); + sha1.update(content2.getBytes(StandardCharsets.UTF_8)); + String hash2 = formatMessageDigest(sha1.digest()); + + Files.write(bagDir.resolve("manifest-sha1.txt"), (hash1 + " data/t1.txt\n" + hash2 + " data/t2.txt\n").getBytes()); + + URL url1 = new URL("http://localhost:" + port + "/url1"); + URL url2 = new URL("http://localhost:" + port + "/url2"); + Files.write(bagDir.resolve("fetch.txt"), (url1 + " " + content1.length() + " data/t1.txt\n" + url2 + " " + content2.length() + " data/t2.txt\n").getBytes()); + + Bag bag = reader.read(bagDir); + + Map> urlConfigs = new HashMap<>(); + Map s1 = new HashMap<>(); s1.put(header1, "v1"); + Map s2 = new HashMap<>(); s2.put(header2, "v2"); + urlConfigs.put(url1.toString(), s1); + urlConfigs.put(url2.toString(), s2); + + sut.isValid(bag, true, true, null, urlConfigs); + + Assertions.assertTrue(h1Received.get(), "Header 1 should have been received"); + Assertions.assertTrue(h2Received.get(), "Header 2 should have been received"); + } + + private String formatMessageDigest(final byte[] digest) { + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } +} diff --git a/src/test/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTaskTest.java b/src/test/java/nl/knaw/dans/bagit/verify/CheckIfFileExistsTaskTest.java similarity index 56% rename from src/test/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTaskTest.java rename to src/test/java/nl/knaw/dans/bagit/verify/CheckIfFileExistsTaskTest.java index eb0f1b7fb..c929c38e1 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTaskTest.java +++ b/src/test/java/nl/knaw/dans/bagit/verify/CheckIfFileExistsTaskTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.nio.file.Path; import java.text.Normalizer; @@ -11,13 +26,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.TempFolderTest; +import nl.knaw.dans.bagit.TempFolderTest; public class CheckIfFileExistsTaskTest extends TempFolderTest { @Test public void testNormalizedFileExists() throws Exception{ - ExecutorService executor = Executors.newCachedThreadPool(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); CountDownLatch latch = new CountDownLatch(1); Set missingFiles = new ConcurrentSkipListSet<>(); String filename = "Núñez.txt"; diff --git a/src/test/java/gov/loc/repository/bagit/verify/FileCountAndTotalSizeVistorTest.java b/src/test/java/nl/knaw/dans/bagit/verify/FileCountAndTotalSizeVistorTest.java similarity index 55% rename from src/test/java/gov/loc/repository/bagit/verify/FileCountAndTotalSizeVistorTest.java rename to src/test/java/nl/knaw/dans/bagit/verify/FileCountAndTotalSizeVistorTest.java index 32ba5a729..1283be4e0 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/FileCountAndTotalSizeVistorTest.java +++ b/src/test/java/nl/knaw/dans/bagit/verify/FileCountAndTotalSizeVistorTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.File; import java.io.IOException; @@ -10,7 +25,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.TestUtils; +import nl.knaw.dans.bagit.TestUtils; /** * Tests the ignore of hidden files while walking the file tree. diff --git a/src/test/java/gov/loc/repository/bagit/verify/MandatoryVerifierTest.java b/src/test/java/nl/knaw/dans/bagit/verify/MandatoryVerifierTest.java similarity index 74% rename from src/test/java/gov/loc/repository/bagit/verify/MandatoryVerifierTest.java rename to src/test/java/nl/knaw/dans/bagit/verify/MandatoryVerifierTest.java index 005a5f414..14b9d3a1c 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/MandatoryVerifierTest.java +++ b/src/test/java/nl/knaw/dans/bagit/verify/MandatoryVerifierTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.File; import java.io.IOException; @@ -10,16 +25,16 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.exceptions.FileNotInPayloadDirectoryException; +import nl.knaw.dans.bagit.exceptions.MissingPayloadDirectoryException; +import nl.knaw.dans.bagit.exceptions.MissingPayloadManifestException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.MissingBagitFileException; -import gov.loc.repository.bagit.exceptions.MissingPayloadDirectoryException; -import gov.loc.repository.bagit.exceptions.MissingPayloadManifestException; -import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.exceptions.MissingBagitFileException; +import nl.knaw.dans.bagit.reader.BagReader; public class MandatoryVerifierTest extends PrivateConstructorTest { @@ -36,7 +51,7 @@ public void testErrorWhenFetchItemsDontExist() throws Exception{ rootDir = Paths.get(new File("src/test/resources/bad-fetch-bag").toURI()); Bag bag = reader.read(rootDir); - Assertions.assertThrows(FileNotInPayloadDirectoryException.class, + Assertions.assertThrows(FileNotInPayloadDirectoryException.class, () -> { MandatoryVerifier.checkFetchItemsExist(bag.getItemsToFetch(), bag.getRootDir()); }); } @@ -47,7 +62,7 @@ public void testErrorWhenMissingPayloadDirectory() throws Exception{ Path dataDir = createDirectory("data"); deleteDirectory(dataDir); - Assertions.assertThrows(MissingPayloadDirectoryException.class, + Assertions.assertThrows(MissingPayloadDirectoryException.class, () -> { MandatoryVerifier.checkPayloadDirectoryExists(bag); }); } @@ -58,7 +73,7 @@ public void testErrorWhenMissingPayloadManifest() throws Exception{ Path manifestFile = folder.resolve("manifest-md5.txt"); Files.delete(manifestFile); - Assertions.assertThrows(MissingPayloadManifestException.class, + Assertions.assertThrows(MissingPayloadManifestException.class, () -> { MandatoryVerifier.checkIfAtLeastOnePayloadManifestsExist(bag.getRootDir(), bag.getVersion()); }); } diff --git a/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java b/src/test/java/nl/knaw/dans/bagit/verify/ManifestVerifierTest.java similarity index 54% rename from src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java rename to src/test/java/nl/knaw/dans/bagit/verify/ManifestVerifierTest.java index 3293b8a79..736ccdd49 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java +++ b/src/test/java/nl/knaw/dans/bagit/verify/ManifestVerifierTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.File; import java.nio.file.Path; @@ -9,22 +24,22 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.exceptions.FileNotInManifestException; -import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException; -import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; -import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.exceptions.FileNotInManifestException; +import nl.knaw.dans.bagit.exceptions.FileNotInPayloadDirectoryException; +import nl.knaw.dans.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.reader.BagReader; -public class PayloadVerifierTest { +public class ManifestVerifierTest { private Path rootDir = Paths.get(new File("src/test/resources/bags/v0_97/bag").toURI()); private BagReader reader = new BagReader(); - private PayloadVerifier sut; + private ManifestVerifier sut; @BeforeEach public void setup(){ - sut = new PayloadVerifier(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); + sut = new ManifestVerifier(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); } @Test @@ -32,11 +47,11 @@ public void testOtherConstructors() throws Exception { rootDir = Paths.get(new File("src/test/resources/bags/v0_96/bag-with-tagfiles-in-payload-manifest").toURI()); Bag bag = reader.read(rootDir); - sut = new PayloadVerifier(); - sut.verifyPayload(bag, true); + sut = new ManifestVerifier(); + sut.verifyManifests(bag, true); - sut = new PayloadVerifier(Executors.newCachedThreadPool()); - sut.verifyPayload(bag, true); + sut = new ManifestVerifier(Executors.newCachedThreadPool()); + sut.verifyManifests(bag, true); } @Test @@ -45,7 +60,7 @@ public void testErrorWhenManifestListFileThatDoesntExist() throws Exception{ Bag bag = reader.read(rootDir); Assertions.assertThrows(FileNotInPayloadDirectoryException.class, - () -> { sut.verifyPayload(bag, true); }); + () -> { sut.verifyManifests(bag, true); }); } @Test @@ -54,7 +69,7 @@ public void testErrorWhenFileIsntInManifest() throws Exception{ Bag bag = reader.read(rootDir); Assertions.assertThrows(FileNotInManifestException.class, - () -> { sut.verifyPayload(bag, true); }); + () -> { sut.verifyManifests(bag, true); }); } @Test @@ -62,7 +77,7 @@ public void testBagWithTagFilesInPayloadIsValid() throws Exception{ rootDir = Paths.get(new File("src/test/resources/bags/v0_96/bag-with-tagfiles-in-payload-manifest").toURI()); Bag bag = reader.read(rootDir); - sut.verifyPayload(bag, true); + sut.verifyManifests(bag, true); } @Test @@ -70,6 +85,6 @@ public void testNotALlFilesListedInAllManifestsThrowsException() throws Exceptio Path bagDir = Paths.get(new File("src/test/resources/notAllFilesListedInAllManifestsBag").toURI()); Bag bag = reader.read(bagDir); Assertions.assertThrows(FileNotInManifestException.class, - () -> { sut.verifyPayload(bag, true); }); + () -> { sut.verifyManifests(bag, true); }); } } diff --git a/src/test/java/nl/knaw/dans/bagit/verify/MySupportedNameToAlgorithmMapping.java b/src/test/java/nl/knaw/dans/bagit/verify/MySupportedNameToAlgorithmMapping.java new file mode 100644 index 000000000..5b33fdc20 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/verify/MySupportedNameToAlgorithmMapping.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import nl.knaw.dans.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; +import nl.knaw.dans.bagit.hash.SupportedAlgorithm; + +public class MySupportedNameToAlgorithmMapping implements BagitAlgorithmNameToSupportedAlgorithmMapping { + + @Override + public SupportedAlgorithm getSupportedAlgorithm(String bagitAlgorithmName) { + if("sha3256".equals(bagitAlgorithmName)){ + return new SHA3256Algorithm(); + } + + return StandardSupportedAlgorithms.valueOf(bagitAlgorithmName.toUpperCase()); + } + +} diff --git a/src/test/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistorTest.java b/src/test/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistorTest.java new file mode 100644 index 000000000..2ff74d6f5 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistorTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import java.util.HashSet; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import nl.knaw.dans.bagit.TempFolderTest; +import nl.knaw.dans.bagit.exceptions.FileNotInManifestException; + +public class PayloadFileExistsInAtLeastOneManifestVistorTest extends TempFolderTest { + + @Test + public void testFileNotInManifestException() throws Exception{ + + PayloadFileExistsInAtLeastOneManifestVistor sut = new PayloadFileExistsInAtLeastOneManifestVistor(new HashSet<>(), true); + Assertions.assertThrows(FileNotInManifestException.class, + () -> { sut.visitFile(createFile("aNewFile"), null); }); + } +} diff --git a/src/test/java/gov/loc/repository/bagit/verify/QuickVerifierTest.java b/src/test/java/nl/knaw/dans/bagit/verify/QuickVerifierTest.java similarity index 72% rename from src/test/java/gov/loc/repository/bagit/verify/QuickVerifierTest.java rename to src/test/java/nl/knaw/dans/bagit/verify/QuickVerifierTest.java index 90b71897c..de8b170fb 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/QuickVerifierTest.java +++ b/src/test/java/nl/knaw/dans/bagit/verify/QuickVerifierTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.verify; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; import java.io.File; import java.lang.reflect.InvocationTargetException; @@ -8,11 +23,11 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.exceptions.InvalidPayloadOxumException; -import gov.loc.repository.bagit.exceptions.PayloadOxumDoesNotExistException; -import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.exceptions.InvalidPayloadOxumException; +import nl.knaw.dans.bagit.exceptions.PayloadOxumDoesNotExistException; +import nl.knaw.dans.bagit.reader.BagReader; public class QuickVerifierTest extends PrivateConstructorTest { diff --git a/src/test/java/nl/knaw/dans/bagit/verify/SHA3256Algorithm.java b/src/test/java/nl/knaw/dans/bagit/verify/SHA3256Algorithm.java new file mode 100644 index 000000000..51ba3f5ba --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/verify/SHA3256Algorithm.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.verify; + +import nl.knaw.dans.bagit.hash.SupportedAlgorithm; + +public class SHA3256Algorithm implements SupportedAlgorithm { + + @Override + public String getMessageDigestName() { + return "SHA3-256"; + } + + @Override + public String getBagitName() { + return "sha3256"; + } + +} diff --git a/src/test/java/gov/loc/repository/bagit/writer/BagWriterTest.java b/src/test/java/nl/knaw/dans/bagit/writer/BagWriterTest.java similarity index 85% rename from src/test/java/gov/loc/repository/bagit/writer/BagWriterTest.java rename to src/test/java/nl/knaw/dans/bagit/writer/BagWriterTest.java index 2b27dcf2e..5ced4be4a 100644 --- a/src/test/java/gov/loc/repository/bagit/writer/BagWriterTest.java +++ b/src/test/java/nl/knaw/dans/bagit/writer/BagWriterTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; @@ -7,15 +22,15 @@ import java.util.Arrays; import java.util.List; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.Bag; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.creator.BagCreator; -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; -import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.bagit.creator.BagCreator; +import nl.knaw.dans.bagit.reader.BagReader; public class BagWriterTest extends PrivateConstructorTest { diff --git a/src/test/java/gov/loc/repository/bagit/writer/BagitFileWriterTest.java b/src/test/java/nl/knaw/dans/bagit/writer/BagitFileWriterTest.java similarity index 63% rename from src/test/java/gov/loc/repository/bagit/writer/BagitFileWriterTest.java rename to src/test/java/nl/knaw/dans/bagit/writer/BagitFileWriterTest.java index 9a2c9ba71..a29530e40 100644 --- a/src/test/java/gov/loc/repository/bagit/writer/BagitFileWriterTest.java +++ b/src/test/java/nl/knaw/dans/bagit/writer/BagitFileWriterTest.java @@ -1,15 +1,30 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import nl.knaw.dans.bagit.PrivateConstructorTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Version; +import nl.knaw.dans.bagit.domain.Version; public class BagitFileWriterTest extends PrivateConstructorTest { diff --git a/src/test/java/gov/loc/repository/bagit/writer/FetchWriterTest.java b/src/test/java/nl/knaw/dans/bagit/writer/FetchWriterTest.java similarity index 79% rename from src/test/java/gov/loc/repository/bagit/writer/FetchWriterTest.java rename to src/test/java/nl/knaw/dans/bagit/writer/FetchWriterTest.java index 144bdf479..000eabc2c 100644 --- a/src/test/java/gov/loc/repository/bagit/writer/FetchWriterTest.java +++ b/src/test/java/nl/knaw/dans/bagit/writer/FetchWriterTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.lang.reflect.InvocationTargetException; import java.net.URL; @@ -9,12 +24,11 @@ import java.util.Arrays; import java.util.List; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.FetchItem; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.FetchItem; - public class FetchWriterTest extends PrivateConstructorTest { @Test diff --git a/src/test/java/gov/loc/repository/bagit/writer/ManifestWriterTest.java b/src/test/java/nl/knaw/dans/bagit/writer/ManifestWriterTest.java similarity index 72% rename from src/test/java/gov/loc/repository/bagit/writer/ManifestWriterTest.java rename to src/test/java/nl/knaw/dans/bagit/writer/ManifestWriterTest.java index 2717d94fb..fea405917 100644 --- a/src/test/java/gov/loc/repository/bagit/writer/ManifestWriterTest.java +++ b/src/test/java/nl/knaw/dans/bagit/writer/ManifestWriterTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -10,13 +25,12 @@ import java.util.List; import java.util.Set; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; - public class ManifestWriterTest extends PrivateConstructorTest { @Test diff --git a/src/test/java/nl/knaw/dans/bagit/writer/MetadataWriterTest.java b/src/test/java/nl/knaw/dans/bagit/writer/MetadataWriterTest.java new file mode 100644 index 000000000..1864a7fa8 --- /dev/null +++ b/src/test/java/nl/knaw/dans/bagit/writer/MetadataWriterTest.java @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import nl.knaw.dans.bagit.PrivateConstructorTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import nl.knaw.dans.bagit.domain.Metadata; +import nl.knaw.dans.bagit.domain.Version; +import nl.knaw.dans.bagit.reader.KeyValueReader; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MetadataWriterTest extends PrivateConstructorTest { + + @Test + public void testClassIsWellDefined() throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException{ + assertUtilityClassWellDefined(MetadataWriter.class); + } + + @Test + public void testWriteBagitInfoFile() throws IOException{ + Path rootDir = createDirectory("writeBagitInfo"); + Path bagInfo = rootDir.resolve("bag-info.txt"); + Path packageInfo = rootDir.resolve("package-info.txt"); + Metadata metadata = new Metadata(); + metadata.add("key1", "value1"); + metadata.add("key2", "value2"); + metadata.add("key3", "value3"); + + Assertions.assertFalse(Files.exists(bagInfo)); + Assertions.assertFalse(Files.exists(packageInfo)); + + MetadataWriter.writeBagMetadata(metadata, new Version(0,96), rootDir, StandardCharsets.UTF_8); + assertTrue(Files.exists(bagInfo)); + + MetadataWriter.writeBagMetadata(metadata, new Version(0,95), rootDir, StandardCharsets.UTF_8); + assertTrue(Files.exists(packageInfo)); + } + + @Test + public void testWriteAndReadMultilineMetadata() throws Exception { + Path rootDir = createDirectory("multilineTest"); + Metadata metadata = new Metadata(); + String multilineValue = "This is a" + System.lineSeparator() + "multi-line" + System.lineSeparator() + "value."; + metadata.add("Description", multilineValue); + metadata.add("Contact-Name", "John Doe"); + + MetadataWriter.writeBagMetadata(metadata, new Version(1, 0), rootDir, StandardCharsets.UTF_8); + + Path bagInfo = rootDir.resolve("bag-info.txt"); + String content = Files.readString(bagInfo, StandardCharsets.UTF_8); + + // Check if subsequent lines are indented + String[] lines = content.split("\\R"); + assertTrue(lines[0].startsWith("Description: ")); + assertTrue(lines[1].startsWith(" "), "Line 2 should be indented"); + assertTrue(lines[2].startsWith(" "), "Line 3 should be indented"); + + // Read it back + List> readMetadata = KeyValueReader.readKeyValuesFromFile(bagInfo, ":", StandardCharsets.UTF_8); + + boolean foundDescription = false; + for (SimpleImmutableEntry entry : readMetadata) { + if ("Description".equals(entry.getKey())) { + Assertions.assertEquals(multilineValue, entry.getValue()); + foundDescription = true; + } + } + assertTrue(foundDescription); + } + + @Test + public void testSanitizeMetadata() throws Exception { + Path rootDir = createDirectory("sanitizeTest"); + Metadata metadata = new Metadata(); + // \u0000 is a non-printable char, \u0007 is bell + String dirtyValue = "Value with\u0000 non-printable\u0007 chars."; + String cleanValue = "Value with non-printable chars."; + metadata.add("Custom-Key", dirtyValue); + + MetadataWriter.writeBagMetadata(metadata, new Version(1, 0), rootDir, StandardCharsets.UTF_8); + + Path bagInfo = rootDir.resolve("bag-info.txt"); + List> readMetadata = KeyValueReader.readKeyValuesFromFile(bagInfo, ":", StandardCharsets.UTF_8); + + boolean found = false; + for (SimpleImmutableEntry entry : readMetadata) { + if ("Custom-Key".equals(entry.getKey())) { + Assertions.assertEquals(cleanValue, entry.getValue()); + found = true; + } + } + assertTrue(found); + } + + @Test + public void testWrapLongLines() throws Exception { + Path rootDir = createDirectory("wrapLongLinesTest"); + Metadata metadata = new Metadata(); + String longValue = "This is a very long value that should definitely exceed the seventy-nine characters limit that is recommended by the BagIt RFC 8493 section two point two point two."; + // Length is ~166 chars. + metadata.add("Long-Key", longValue); + + MetadataWriter.writeBagMetadata(metadata, new Version(1, 0), rootDir, StandardCharsets.UTF_8); + + Path bagInfo = rootDir.resolve("bag-info.txt"); + String content = Files.readString(bagInfo, StandardCharsets.UTF_8); + + String[] lines = content.split("\\R"); + for (String line : lines) { + assertTrue(line.length() <= 79, "Line length should be <= 79: " + line.length()); + if (!line.startsWith("Long-Key: ")) { + assertTrue(line.startsWith(" "), "Wrapped lines should be indented"); + } + } + + // Read it back + List> readMetadata = KeyValueReader.readKeyValuesFromFile(bagInfo, ":", StandardCharsets.UTF_8); + boolean foundLongKey = false; + for (SimpleImmutableEntry entry : readMetadata) { + if ("Long-Key".equals(entry.getKey())) { + // Since it's wrapped at spaces, the space is replaced by newline in reading if it's joined by newline + // But wrapLine also preserves spaces? Let's check what it actually produces. + // It should match the longValue with some spaces replaced by newlines. + Assertions.assertEquals(longValue, entry.getValue().replace(System.lineSeparator(), " ")); + foundLongKey = true; + } + } + assertTrue(foundLongKey); + } + + @Test + public void testMultilineAndWrapping() throws Exception { + Path rootDir = createDirectory("multilineAndWrappingTest"); + Metadata metadata = new Metadata(); + // A multiline value where the first line is short and the second is long + String value = "Short first line\nThis is a very long second line that will definitely need wrapping because it's much longer than seventy-nine characters."; + metadata.add("Description", value); + + MetadataWriter.writeBagMetadata(metadata, new Version(1, 0), rootDir, StandardCharsets.UTF_8); + + Path bagInfo = rootDir.resolve("bag-info.txt"); + String content = Files.readString(bagInfo, StandardCharsets.UTF_8); + + // Check for double newlines or lines containing only a space + String[] lines = content.split("\\r?\\n"); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + Assertions.assertFalse(line.isEmpty() && i < lines.length - 1, "Should not have empty lines in the middle (caused by double newlines)"); + if (i > 0) { + // All lines after the first one of an entry must be indented + assertTrue(line.startsWith(" "), "Continuation line must start with a space: [" + line + "]"); + // If it was a double newline, we'd see a line that is just " " followed by another line + Assertions.assertNotEquals(" ", line, "Should not have a line that is just a single space"); + } + } + } +} diff --git a/src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java b/src/test/java/nl/knaw/dans/bagit/writer/PayloadWriterTest.java similarity index 74% rename from src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java rename to src/test/java/nl/knaw/dans/bagit/writer/PayloadWriterTest.java index fd23fb768..87f25e6cb 100644 --- a/src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java +++ b/src/test/java/nl/knaw/dans/bagit/writer/PayloadWriterTest.java @@ -1,4 +1,19 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -11,14 +26,13 @@ import java.util.HashSet; import java.util.Set; +import nl.knaw.dans.bagit.PrivateConstructorTest; +import nl.knaw.dans.bagit.domain.FetchItem; +import nl.knaw.dans.bagit.domain.Manifest; +import nl.knaw.dans.bagit.hash.StandardSupportedAlgorithms; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; -import gov.loc.repository.bagit.domain.FetchItem; -import gov.loc.repository.bagit.domain.Manifest; -import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; - public class PayloadWriterTest extends PrivateConstructorTest { @Test @@ -55,7 +69,7 @@ public void testWritePayloadFilesMinusFetchFiles() throws IOException, URISyntax Assertions.assertFalse(Files.exists(copiedFile) || Files.exists(copiedFile.getParent())); PayloadWriter.writePayloadFiles(payloadManifests, - Arrays.asList(new FetchItem(null, null, Paths.get("data/dir1/test3.txt"))), + Arrays.asList(new FetchItem(null, null, Paths.get("data/dir1/test3.txt"))), outputDir, rootDir.resolve("data")); Assertions.assertFalse(Files.exists(copiedFile) || Files.exists(copiedFile.getParent())); diff --git a/src/test/java/gov/loc/repository/bagit/writer/RelativePathWriterTest.java b/src/test/java/nl/knaw/dans/bagit/writer/RelativePathWriterTest.java similarity index 60% rename from src/test/java/gov/loc/repository/bagit/writer/RelativePathWriterTest.java rename to src/test/java/nl/knaw/dans/bagit/writer/RelativePathWriterTest.java index d9c83dccb..798bc9fcd 100644 --- a/src/test/java/gov/loc/repository/bagit/writer/RelativePathWriterTest.java +++ b/src/test/java/nl/knaw/dans/bagit/writer/RelativePathWriterTest.java @@ -1,14 +1,28 @@ -package gov.loc.repository.bagit.writer; +/* + * Copyright (C) 2023 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.bagit.writer; import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; +import nl.knaw.dans.bagit.PrivateConstructorTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import gov.loc.repository.bagit.PrivateConstructorTest; - public class RelativePathWriterTest extends PrivateConstructorTest { @Test public void testClassIsWellDefined() throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException{ diff --git a/src/test/resources/bagitProfiles/exampleProfile.json b/src/test/resources/bagitProfiles/exampleProfile.json index a86d1a74e..6a7db5135 100644 --- a/src/test/resources/bagitProfiles/exampleProfile.json +++ b/src/test/resources/bagitProfiles/exampleProfile.json @@ -1,85 +1,99 @@ { - "BagIt-Profile-Info":{ - "BagIt-Profile-Identifier":"http://canadiana.org/standards/bagit/tdr_ingest.json", - "Source-Organization":"Candiana.org", - "Contact-Name":"William Wueppelmann", - "Contact-Email":"tdr@canadiana.com", - "External-Description":"BagIt profile for ingesting content into the C.O. TDR loading dock.", - "Version":"1.2" - }, - "Bag-Info":{ - "Source-Organization":{ - "required":true, - "values":[ - "Simon Fraser University", - "York University" - ] - }, - "Organization-Address":{ - "required":true, - "values":[ - "8888 University Drive Burnaby, B.C. V5A 1S6 Canada", - "4700 Keele Street Toronto, Ontario M3J 1P3 Canada" - ] - }, - "Contact-Name":{ - "required":true, - "values":[ - "Mark Jordan", - "Nick Ruest" - ] - }, - "Contact-Phone":{ - "required":false - }, - "Contact-Email":{ - "required":true - }, - "External-Description":{ - "required":true - }, - "External-Identifier":{ - "required":false - }, - "Bag-Size":{ - "required":true - }, - "Bag-Group-Identifier":{ - "required":false - }, - "Bag-Count":{ - "required":true - }, - "Internal-Sender-Identifier":{ - "required":false - }, - "Internal-Sender-Description":{ - "required":false - }, - "Bagging-Date":{ - "required":true, - "repeatable": false - }, - "Payload-Oxum":{ - "required":true - } - }, - "Manifests-Required":[ - "md5" - ], - "Allow-Fetch.txt":false, - "Serialization":"forbidden", - "Accept-Serialization":[ - "application/zip" - ], - "Tag-Manifests-Required":[ - "md5" - ], - "Tag-Files-Required":[ - "DPN/dpnFirstNode.txt", - "DPN/dpnRegistry" - ], - "Accept-BagIt-Version":[ - "0.96" - ] + "BagIt-Profile-Info": { + "BagIt-Profile-Identifier": "http://canadiana.org/standards/bagit/tdr_ingest.json", + "Source-Organization": "Candiana.org", + "Contact-Name": "William Wueppelmann", + "Contact-Email": "tdr@canadiana.com", + "Contact-Phone": "+1 613 907 7040", + "External-Description": "BagIt profile for ingesting content into the C.O. TDR loading dock.", + "Version": "1.2" + }, + "Bag-Info": { + "Source-Organization": { + "required": true, + "values": [ + "Simon Fraser University", + "York University" + ], + "repeatable": false + }, + "Organization-Address": { + "required": true, + "values": [ + "8888 University Drive Burnaby, B.C. V5A 1S6 Canada", + "4700 Keele Street Toronto, Ontario M3J 1P3 Canada" + ], + "repeatable": false + }, + "Contact-Name": { + "required": true, + "values": [ + "Mark Jordan", + "Nick Ruest" + ], + "repeatable": false + }, + "Contact-Phone": { + "required": false, + "repeatable": false + }, + "Contact-Email": { + "required": true, + "repeatable": false + }, + "External-Description": { + "required": true, + "repeatable": false + }, + "External-Identifier": { + "required": false, + "repeatable": false + }, + "Bag-Size": { + "required": true, + "repeatable": false + }, + "Bag-Group-Identifier": { + "required": false, + "repeatable": false + }, + "Bag-Count": { + "required": true, + "repeatable": false + }, + "Internal-Sender-Identifier": { + "required": false, + "repeatable": false + }, + "Internal-Sender-Description": { + "required": false, + "repeatable": false + }, + "Bagging-Date": { + "required": true, + "repeatable": false + }, + "Payload-Oxum": { + "required": true, + "repeatable": false + } + }, + "Manifests-Required": [ + "md5" + ], + "Allow-Fetch.txt": false, + "Serialization": "forbidden", + "Accept-Serialization": [ + "application/zip" + ], + "Tag-Manifests-Required": [ + "md5" + ], + "Tag-Files-Required": [ + "DPN/dpnFirstNode.txt", + "DPN/dpnRegistry" + ], + "Accept-BagIt-Version": [ + "0.96" + ] } \ No newline at end of file diff --git a/src/test/resources/bagitProfiles/exampleProfileOnlyRequiredFields.json b/src/test/resources/bagitProfiles/exampleProfileOnlyRequiredFields.json new file mode 100644 index 000000000..dead840fb --- /dev/null +++ b/src/test/resources/bagitProfiles/exampleProfileOnlyRequiredFields.json @@ -0,0 +1,84 @@ +{ + "BagIt-Profile-Info": { + "BagIt-Profile-Identifier": "http://canadiana.org/standards/bagit/tdr_ingest.json", + "Source-Organization": "Candiana.org", + "External-Description": "BagIt profile for ingesting content into the C.O. TDR loading dock.", + "Version": "1.2" + }, + "Bag-Info": { + "Source-Organization": { + "required": true, + "values": [ + "Simon Fraser University", + "York University" + ], + "repeatable": false + }, + "Organization-Address": { + "required": true, + "values": [ + "8888 University Drive Burnaby, B.C. V5A 1S6 Canada", + "4700 Keele Street Toronto, Ontario M3J 1P3 Canada" + ], + "repeatable": false + }, + "Contact-Name": { + "required": true, + "values": [ + "Mark Jordan", + "Nick Ruest" + ], + "repeatable": false + }, + "Contact-Phone": { + "repeatable": false + }, + "Contact-Email": { + "required": true, + "repeatable": false + }, + "External-Description": { + "required": true, + "repeatable": false + }, + "External-Identifier": { + "repeatable": false + }, + "Bag-Size": { + "required": true, + "repeatable": false + }, + "Bag-Group-Identifier": { + "repeatable": false + }, + "Bag-Count": { + "required": true, + "repeatable": false + }, + "Internal-Sender-Identifier": { + "repeatable": false + }, + "Internal-Sender-Description": { + "repeatable": false + }, + "Bagging-Date": { + "required": true, + "repeatable": false + }, + "Payload-Oxum": { + "required": true, + "repeatable": false + } + }, + "Manifests-Required": [ + "md5" + ], + "Allow-Fetch.txt": false, + "Serialization": "forbidden", + "Accept-Serialization": [ + "application/zip" + ], + "Accept-BagIt-Version": [ + "0.96" + ] +} \ No newline at end of file diff --git a/src/test/resources/bags/v1_0/bag/bag-info.txt b/src/test/resources/bags/v1_0/bag/bag-info.txt new file mode 100644 index 000000000..30723bdc8 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-22 +Payload-Oxum: 6.1 diff --git a/src/test/resources/bags/v1_0/bag/bagit.txt b/src/test/resources/bags/v1_0/bag/bagit.txt new file mode 100644 index 000000000..7bfcaecc4 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 1.0 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/bags/v1_0/bag/data/foo.txt b/src/test/resources/bags/v1_0/bag/data/foo.txt new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/data/foo.txt @@ -0,0 +1 @@ +hello diff --git a/src/test/resources/bags/v1_0/bag/manifest-sha512.txt b/src/test/resources/bags/v1_0/bag/manifest-sha512.txt new file mode 100644 index 000000000..f3fd06d72 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/manifest-sha512.txt @@ -0,0 +1 @@ +e7c22b994c59d9cf2b48e549b1e24666636045930d3da7c1acb299d1c3b7f931f94aae41edda2c2b207a36e10f8bcb8d45223e54878f5b316e7ce3b6bc019629 data/foo.txt diff --git a/src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt b/src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt new file mode 100644 index 000000000..732e7c45d --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt @@ -0,0 +1,3 @@ +f5bdbc7f273dd8b95d30b77b3d6f727d999ecbe9be06a7656388e7a3a46d963881563d779a2a99265b0c2785de2a7b72ac05fa7bc5b66b471d3f4e80fe9bb370 bag-info.txt +1d73ae108d4109b61f56698a5e19ee1f8947bdf8940bbce6adbe5e0940c2363caace6a547b4f1b3ec6a4fd2b7fa845e9cb9d28823bc72c59971718bb26f2fbd8 bagit.txt +35d40e38f5e2eb7261a1cc0c0ccf9c6c50d2e07e39a9460ec4f99046048f939d82647c68d8f744c653c7a37c05cc9d71153c8028786dfeba4322a3be5fe81af0 manifest-sha512.txt diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml new file mode 100644 index 000000000..2fa3d828d --- /dev/null +++ b/src/test/resources/logback.xml @@ -0,0 +1,16 @@ + + + + + %-5level %msg%n + + + + + + + + + + + diff --git a/src/test/resources/md5Bag/bag-info.txt b/src/test/resources/md5Bag/bag-info.txt new file mode 100644 index 000000000..7dd5e14ce --- /dev/null +++ b/src/test/resources/md5Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 52.1 diff --git a/src/test/resources/md5Bag/bagit.txt b/src/test/resources/md5Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/md5Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/md5Bag/data/readme.txt b/src/test/resources/md5Bag/data/readme.txt new file mode 100644 index 000000000..7ec7b2d61 --- /dev/null +++ b/src/test/resources/md5Bag/data/readme.txt @@ -0,0 +1 @@ +this is a valid md5 bag used for testing valid bags diff --git a/src/test/resources/md5Bag/manifest-md5.txt b/src/test/resources/md5Bag/manifest-md5.txt new file mode 100644 index 000000000..62cd3ebf9 --- /dev/null +++ b/src/test/resources/md5Bag/manifest-md5.txt @@ -0,0 +1 @@ +aee452eebfbd978228775bf7b0e808dc data/readme.txt diff --git a/src/test/resources/md5Bag/tagmanifest-md5.txt b/src/test/resources/md5Bag/tagmanifest-md5.txt new file mode 100644 index 000000000..6201d4ecf --- /dev/null +++ b/src/test/resources/md5Bag/tagmanifest-md5.txt @@ -0,0 +1,3 @@ +59044a83bd40ec50b7ee927bf1e672a9 bag-info.txt +9e5ad981e0d29adc278f6a294b8c2aca bagit.txt +f1578aeea7f57af650f0a94565e40ceb manifest-md5.txt diff --git a/src/test/resources/sha1Bag/bag-info.txt b/src/test/resources/sha1Bag/bag-info.txt new file mode 100644 index 000000000..30ce5278d --- /dev/null +++ b/src/test/resources/sha1Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 43.1 diff --git a/src/test/resources/sha1Bag/bagit.txt b/src/test/resources/sha1Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha1Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha1Bag/data/readme.txt b/src/test/resources/sha1Bag/data/readme.txt new file mode 100644 index 000000000..0979f66d0 --- /dev/null +++ b/src/test/resources/sha1Bag/data/readme.txt @@ -0,0 +1 @@ +this is a sha1 bag used to test valid bags diff --git a/src/test/resources/sha1Bag/manifest-sha1.txt b/src/test/resources/sha1Bag/manifest-sha1.txt new file mode 100644 index 000000000..04bda92f9 --- /dev/null +++ b/src/test/resources/sha1Bag/manifest-sha1.txt @@ -0,0 +1 @@ +2f0973b52379ea88967e5e319941ff167d680ee9 data/readme.txt diff --git a/src/test/resources/sha1Bag/tagmanifest-sha1.txt b/src/test/resources/sha1Bag/tagmanifest-sha1.txt new file mode 100644 index 000000000..f4d7d5534 --- /dev/null +++ b/src/test/resources/sha1Bag/tagmanifest-sha1.txt @@ -0,0 +1,3 @@ +27bfaef124e06412f40c75bf6e5aa1cf4dbb9774 bag-info.txt +e2924b081506bac23f5fffe650ad1848a1c8ac1d bagit.txt +54adcabf9b1038d090d145d9315070c5493db1e6 manifest-sha1.txt diff --git a/src/test/resources/sha224Bag/bag-info.txt b/src/test/resources/sha224Bag/bag-info.txt new file mode 100644 index 000000000..40c3b6f85 --- /dev/null +++ b/src/test/resources/sha224Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 45.1 diff --git a/src/test/resources/sha224Bag/bagit.txt b/src/test/resources/sha224Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha224Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha224Bag/data/readme.txt b/src/test/resources/sha224Bag/data/readme.txt new file mode 100644 index 000000000..e05de9a0a --- /dev/null +++ b/src/test/resources/sha224Bag/data/readme.txt @@ -0,0 +1 @@ +this is a sha224 valid bag, used for testing diff --git a/src/test/resources/sha224Bag/manifest-sha224.txt b/src/test/resources/sha224Bag/manifest-sha224.txt new file mode 100644 index 000000000..a8ea47300 --- /dev/null +++ b/src/test/resources/sha224Bag/manifest-sha224.txt @@ -0,0 +1 @@ +476ba7f2f92170790679502b1f1bad48a9d0ae74b561cf558370a5fb data/readme.txt diff --git a/src/test/resources/sha224Bag/tagmanifest-sha224.txt b/src/test/resources/sha224Bag/tagmanifest-sha224.txt new file mode 100644 index 000000000..cba870c7c --- /dev/null +++ b/src/test/resources/sha224Bag/tagmanifest-sha224.txt @@ -0,0 +1,3 @@ +34458393a69d3c6a5ec82743e1817032b3dc875ae10ff19b2c55ed1c bag-info.txt +fe7e72d15c56a8e8017246f52765a93ce4823d1fa3ed1e0d1ac3695f bagit.txt +8c717410c037477322cdd4069feeff2aa09eb0991989807724defaae manifest-sha224.txt diff --git a/src/test/resources/sha256Bag/bag-info.txt b/src/test/resources/sha256Bag/bag-info.txt new file mode 100644 index 000000000..ec9f2e4da --- /dev/null +++ b/src/test/resources/sha256Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 44.1 diff --git a/src/test/resources/sha256Bag/bagit.txt b/src/test/resources/sha256Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha256Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha256Bag/data/readme.txt b/src/test/resources/sha256Bag/data/readme.txt new file mode 100644 index 000000000..4e8750d05 --- /dev/null +++ b/src/test/resources/sha256Bag/data/readme.txt @@ -0,0 +1 @@ +this is a valid sha256 bag used for testing diff --git a/src/test/resources/sha256Bag/manifest-sha256.txt b/src/test/resources/sha256Bag/manifest-sha256.txt new file mode 100644 index 000000000..f19ae2b49 --- /dev/null +++ b/src/test/resources/sha256Bag/manifest-sha256.txt @@ -0,0 +1 @@ +694d56447888829536b5782536f2bbe5c83325b1b2fd56a5ca676c991f196bf7 data/readme.txt diff --git a/src/test/resources/sha256Bag/tagmanifest-sha256.txt b/src/test/resources/sha256Bag/tagmanifest-sha256.txt new file mode 100644 index 000000000..c37479e35 --- /dev/null +++ b/src/test/resources/sha256Bag/tagmanifest-sha256.txt @@ -0,0 +1,3 @@ +d0d7117f5662fe588aab40c61c66163e961c930ffec5fd52d64d39a6aa8f8e38 bag-info.txt +e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt +ba57368e0165a727b0b6cc5f040c5cbd3e3ac28d1d52f94d111d64382c6f0466 manifest-sha256.txt diff --git a/src/test/resources/sha512Bag/bag-info.txt b/src/test/resources/sha512Bag/bag-info.txt new file mode 100644 index 000000000..ec9f2e4da --- /dev/null +++ b/src/test/resources/sha512Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 44.1 diff --git a/src/test/resources/sha512Bag/bagit.txt b/src/test/resources/sha512Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha512Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha512Bag/data/readme.txt b/src/test/resources/sha512Bag/data/readme.txt new file mode 100644 index 000000000..218b84225 --- /dev/null +++ b/src/test/resources/sha512Bag/data/readme.txt @@ -0,0 +1 @@ +this is a valid sha512 bag used for testing diff --git a/src/test/resources/sha512Bag/manifest-sha512.txt b/src/test/resources/sha512Bag/manifest-sha512.txt new file mode 100644 index 000000000..d6f3bc3dd --- /dev/null +++ b/src/test/resources/sha512Bag/manifest-sha512.txt @@ -0,0 +1 @@ +6e464149951ad3e3975c326ccb5c6e4ce306516fcc99689d90094e3bfbd87e87e0f7dbb4efd8e59456b808ca2b7a22bb14fa0b1edb76e11964a181850b45a628 data/readme.txt diff --git a/src/test/resources/sha512Bag/tagmanifest-sha512.txt b/src/test/resources/sha512Bag/tagmanifest-sha512.txt new file mode 100644 index 000000000..f9bf3e71a --- /dev/null +++ b/src/test/resources/sha512Bag/tagmanifest-sha512.txt @@ -0,0 +1,3 @@ +bede8e68c6fef4be00b54e5a5c2e104501444531679948187926a84a4e5c85d466959f34dae9ac4c81ccea4ec28e3f0ea7b661c0f100e457b3a568a08661cc01 bag-info.txt +418dcfbe17d5f4b454b18630be795462cf7da4ceb6313afa49451aa2568e41f7ca3d34cf0280c7d056dc5681a70c37586aa1755620520b9198eede905ba2d0f6 bagit.txt +82e79660ff0335b94f56a0b837b1564291b3eb2242a8c28c51d32efc6db9db5ab6fe7656d975bf16989c67ef3a3fa5cbd066eb1bcf422207ec41a20ed0f98943 manifest-sha512.txt