diff --git a/.github/workflows/garm.yml b/.github/workflows/garm.yml index 7487e4033f00..6cb04a9d51d1 100644 --- a/.github/workflows/garm.yml +++ b/.github/workflows/garm.yml @@ -17,7 +17,7 @@ concurrency: jobs: test: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 services: server: image: ${{ matrix.server == 'stable22' && 'ghcr.io/nextcloud/continuous-integration-shallow-server-php8.0:1' || 'ghcr.io/nextcloud/continuous-integration-shallow-server-php8.3:latest' }} @@ -32,6 +32,12 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Free disk space + run: | + df -h / + sudo rm -rf /usr/share/dotnet /usr/share/swift /usr/local/.ghcup /opt/ghc /usr/local/share/boost /usr/local/share/powershell /usr/local/share/chromium /usr/local/julia* /opt/az /opt/microsoft /opt/google + df -h / + - name: configure server run: | sudo apt-get update && sudo apt-get install unzip cpu-checker qemu-system-x86 --yes @@ -43,6 +49,7 @@ jobs: docker exec server chmod +x /tmp/configNC.sh docker exec --user www-data server /tmp/configNC.sh ${{ matrix.server }} docker exec server /usr/local/bin/run.sh + sudo apt-get clean - name: set up JDK 21 uses: actions/setup-java@77dd5b2ce225ea11469928ac317850245a74ae78 # v6.2.0 @@ -50,12 +57,17 @@ jobs: distribution: "temurin" java-version: 21 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@8da46b539cd8fe6ab27ca75128919a18d757604c # v6.4.1 + - name: Configure gradle daemon run: | mkdir -p $HOME/.gradle - echo "org.gradle.jvmargs=-Xmx6g -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 -XX:+UseParallelGC -XX:MaxMetaspaceSize=1g" >> $HOME/.gradle/gradle.properties - echo "org.gradle.caching=true" >> $HOME/.gradle/gradle.properties - echo "org.gradle.parallel=true" >> $HOME/.gradle/gradle.properties + echo "org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=768m -XX:+UseSerialGC -Dfile.encoding=UTF-8 --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.nio.channels=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED" >> $HOME/.gradle/gradle.properties + echo "kotlin.compiler.execution.strategy=in-process" >> $HOME/.gradle/gradle.properties + echo "org.gradle.workers.max=4" >> $HOME/.gradle/gradle.properties + echo "org.gradle.caching=false" >> $HOME/.gradle/gradle.properties + echo "org.gradle.parallel=false" >> $HOME/.gradle/gradle.properties echo "org.gradle.configureondemand=true" >> $HOME/.gradle/gradle.properties - name: Config gplay diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 8ad94b13f8e8..8a612f4f6ec3 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -17,6 +17,9 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget val shotTest = System.getenv("SHOT_TEST") == "true" val ciBuild = System.getenv("CI") == "true" val perfAnalysis = project.hasProperty("perfAnalysis") +val instrumentedTestBuild = project.hasProperty("coverage") || shotTest || + gradle.startParameter.taskNames.any { it.contains("AndroidTest", ignoreCase = true) } +val enableR8 = !instrumentedTestBuild val allAbis = listOf("armeabi-v7a", "arm64-v8a", "x86", "x86_64") val localDebugAbis = when (System.getProperty("os.arch")) { @@ -138,7 +141,7 @@ android { enableAndroidTestCoverage = project.hasProperty("coverage") resConfigs("xxxhdpi") optimization { - enable = true + enable = enableR8 } ndk { abiFilters += if (ciBuild) allAbis else localDebugAbis @@ -169,7 +172,7 @@ android { versionCode = 20220322 versionName = "20220322" optimization { - enable = true + enable = enableR8 } } diff --git a/app/src/androidTest/java/com/owncloud/android/AbstractOnServerIT.kt b/app/src/androidTest/java/com/owncloud/android/AbstractOnServerIT.kt index 9ddaa63b2922..e85045b0052a 100644 --- a/app/src/androidTest/java/com/owncloud/android/AbstractOnServerIT.kt +++ b/app/src/androidTest/java/com/owncloud/android/AbstractOnServerIT.kt @@ -32,6 +32,7 @@ import com.owncloud.android.utils.MimeType import org.apache.commons.httpclient.HttpStatus import org.apache.commons.httpclient.methods.GetMethod import org.junit.After +import org.junit.Assert.assertFalse import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.BeforeClass @@ -92,6 +93,8 @@ abstract class AbstractOnServerIT : AbstractIT() { private const val SERVER_READY_RETRIES = 3 private const val SERVER_READY_RETRY_DELAY_MILLIS = 60_000L + private var isServerUnreachable = false + private val FOLDER_MIME_TYPES = setOf(MimeType.DIRECTORY, MimeType.WEBDAV_FOLDER) private val LOCAL_COPY_BEHAVIOURS = setOf( FileUploadWorker.LOCAL_BEHAVIOUR_COPY, @@ -179,6 +182,7 @@ abstract class AbstractOnServerIT : AbstractIT() { } private fun waitForServer(client: OwnCloudClient, baseUrl: Uri) { + assertFalse("Server not ready! Already unreachable for a previous test class", isServerUnreachable) val statusRequest = GetMethod("$baseUrl/status.php") repeat(SERVER_READY_RETRIES) { @@ -187,7 +191,8 @@ abstract class AbstractOnServerIT : AbstractIT() { sleepIgnoringInterrupt(SERVER_READY_RETRY_DELAY_MILLIS) } - assertTrue("Server not ready!", isServerReady(client, statusRequest)) + isServerUnreachable = !isServerReady(client, statusRequest) + assertFalse("Server not ready!", isServerUnreachable) } private fun isServerReady(client: OwnCloudClient, statusRequest: GetMethod): Boolean = try { diff --git a/app/src/androidTest/java/com/owncloud/android/files/services/FileUploaderIT.kt b/app/src/androidTest/java/com/owncloud/android/files/services/FileUploaderIT.kt index 8c0cb0f72cd2..787eca1b1778 100644 --- a/app/src/androidTest/java/com/owncloud/android/files/services/FileUploaderIT.kt +++ b/app/src/androidTest/java/com/owncloud/android/files/services/FileUploaderIT.kt @@ -46,7 +46,8 @@ class FileUploaderIT : AbstractOnServerIT() { uploadOriginalFile(originalFile) val replacement = getDummyFile(EMPTY_FILE) - assertTrue(uploadOperation(replacement, NameCollisionPolicy.OVERWRITE).execute(client).isSuccess) + val result = uploadOperation(replacement, NameCollisionPolicy.OVERWRITE).execute(client) + assertTrue(result.toString(), result.isSuccess) assertRemoteFileLength(REMOTE_PATH, EMPTY_LENGTH) } diff --git a/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/DocumentsProviderUtils.kt b/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/DocumentsProviderUtils.kt deleted file mode 100644 index 07c337223c89..000000000000 --- a/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/DocumentsProviderUtils.kt +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Nextcloud - Android Client - * - * SPDX-FileCopyrightText: 2020 Torsten Grote - * SPDX-License-Identifier: AGPL-3.0-or-later OR GPL-2.0-only - */ -package com.owncloud.android.providers.documentstorage - -import android.content.Context -import android.database.ContentObserver -import android.database.Cursor -import android.net.Uri -import android.provider.DocumentsContract.Document.COLUMN_DOCUMENT_ID -import android.provider.DocumentsContract.Document.COLUMN_MIME_TYPE -import android.provider.DocumentsContract.Document.MIME_TYPE_DIR -import android.provider.DocumentsContract.EXTRA_LOADING -import android.provider.DocumentsContract.buildChildDocumentsUriUsingTree -import android.provider.DocumentsContract.buildDocumentUriUsingTree -import android.provider.DocumentsContract.buildTreeDocumentUri -import android.provider.DocumentsContract.getDocumentId -import androidx.annotation.VisibleForTesting -import androidx.documentfile.provider.DocumentFile -import com.owncloud.android.datamodel.FileDataStorageManager -import com.owncloud.android.datamodel.OCFile -import com.owncloud.android.lib.common.OwnCloudClient -import com.owncloud.android.lib.common.utils.Log_OC -import com.owncloud.android.lib.resources.files.ExistenceCheckRemoteOperation -import com.owncloud.android.providers.documentstorage.DocumentsStorageProvider.Companion.DOCUMENTID_SEPARATOR -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout -import org.junit.Assert.assertArrayEquals -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNotNull -import org.junit.Assert.assertTrue -import java.io.IOException -import java.io.InputStream -import kotlin.coroutines.resume - -// Uploads can sometimes take a bit of time, so 15sec is still considered recent enough -private const val RECENT_MILLISECONDS = 15_000 - -object DocumentsProviderUtils { - - internal fun DocumentFile.getOCFile(storageManager: FileDataStorageManager): OCFile? { - val id = getDocumentId(uri) - val separated: List = id.split(DOCUMENTID_SEPARATOR.toRegex()) - return storageManager.getFileById(separated[1].toLong()) - } - - internal fun DocumentFile.assertRegularFile( - name: String? = null, - size: Long? = null, - mimeType: String? = null, - parent: DocumentFile? = null - ) { - name?.let { assertEquals(it, this.name) } - assertTrue(exists()) - assertTrue(isFile) - assertFalse(isDirectory) - assertFalse(isVirtual) - size?.let { assertEquals(it, length()) } - mimeType?.let { assertEquals(it, type) } - parent?.let { assertEquals(it.uri.toString(), parentFile!!.uri.toString()) } - } - - internal fun DocumentFile.assertRegularFolder(name: String? = null, parent: DocumentFile? = null) { - name?.let { assertEquals(it, this.name) } - assertTrue(exists()) - assertFalse(isFile) - assertTrue(isDirectory) - assertFalse(isVirtual) - parent?.let { assertEquals(it.uri.toString(), parentFile!!.uri.toString()) } - } - - internal fun DocumentFile.assertRecentlyModified() { - val diff = System.currentTimeMillis() - lastModified() - assertTrue("File $name older than expected: $diff", diff < RECENT_MILLISECONDS) - } - - internal fun assertExistsOnServer(client: OwnCloudClient, remotePath: String, shouldExist: Boolean) { - val result = ExistenceCheckRemoteOperation(remotePath, !shouldExist).execute(client) - assertTrue("$result", result.isSuccess) - } - - internal fun assertListFilesEquals(expected: Collection, actual: Collection) { -// assertEquals( -// "Actual: ${actual.map { it.name.toString() }}", -// expected.map { it.uri.toString() }.apply { sorted() }, -// actual.map { it.uri.toString() }.apply { sorted() }, -// ) - // FIXME replace with commented out stronger assertion above - // when parallel [UploadFileOperation]s don't bring back deleted files - val expectedSet = HashSet(expected.map { it.uri.toString() }) - val actualSet = HashSet(actual.map { it.uri.toString() }) - assertTrue(actualSet.containsAll(expectedSet)) - actualSet.removeAll(expectedSet) - actualSet.forEach { - Log_OC.e("TEST", "Error: Found unexpected file: $it") - } - } - - internal fun assertReadEquals(data: ByteArray, inputStream: InputStream?) { - assertNotNull(inputStream) - inputStream!!.use { - assertArrayEquals(data, it.readBytes()) - } - } - - /** - * Same as [DocumentFile.findFile] only that it re-queries when the first result was stale. - * - * Most documents providers including Nextcloud are listing the full directory content - * when querying for a specific file in a directory, - * so there is no point in trying to optimize the query by not listing all children. - */ - suspend fun DocumentFile.findFileBlocking(context: Context, displayName: String): DocumentFile? { - val files = listFilesBlocking(context) - for (doc in files) { - if (displayName == doc.name) return doc - } - return null - } - - /** - * Works like [DocumentFile.listFiles] except that it waits until the DocumentProvider has a result. - * This prevents getting an empty list even though there are children to be listed. - */ - suspend fun DocumentFile.listFilesBlocking(context: Context) = withContext(Dispatchers.IO) { - val resolver = context.contentResolver - val childrenUri = buildChildDocumentsUriUsingTree(uri, getDocumentId(uri)) - val projection = arrayOf(COLUMN_DOCUMENT_ID, COLUMN_MIME_TYPE) - val result = ArrayList() - - try { - getLoadedCursor { - resolver.query(childrenUri, projection, null, null, null) - } - } catch (e: TimeoutCancellationException) { - throw IOException(e) - }.use { cursor -> - while (cursor.moveToNext()) { - val documentId = cursor.getString(0) - val isDirectory = cursor.getString(1) == MIME_TYPE_DIR - val file = if (isDirectory) { - val treeUri = buildTreeDocumentUri(uri.authority, documentId) - DocumentFile.fromTreeUri(context, treeUri)!! - } else { - val documentUri = buildDocumentUriUsingTree(uri, documentId) - DocumentFile.fromSingleUri(context, documentUri)!! - } - result.add(file) - } - } - result - } - - /** - * Returns a cursor for the given query while ensuring that the cursor was loaded. - * - * When the SAF backend is a cloud storage provider (e.g. Nextcloud), - * it can happen that the query returns an outdated (e.g. empty) cursor - * which will only be updated in response to this query. - * - * See: https://commonsware.com/blog/2019/12/14/scoped-storage-stories-listfiles-woe.html - * - * This method uses a [suspendCancellableCoroutine] to wait for the result of a [ContentObserver] - * registered on the cursor in case the cursor is still loading ([EXTRA_LOADING]). - * If the cursor is not loading, it will be returned right away. - * - * @param timeout an optional time-out in milliseconds - * @throws TimeoutCancellationException if there was no result before the time-out - * @throws IOException if the query returns null - */ - @Suppress("EXPERIMENTAL_API_USAGE") - @VisibleForTesting - internal suspend fun getLoadedCursor(timeout: Long = 15_000, query: () -> Cursor?) = withTimeout(timeout) { - suspendCancellableCoroutine { cont -> - val cursor = query() ?: throw IOException("Initial query returned no results") - cont.invokeOnCancellation { cursor.close() } - val loading = cursor.extras?.getBoolean(EXTRA_LOADING, false) ?: false - if (loading) { - Log_OC.e("TEST", "Cursor was loading, wait for update...") - cursor.registerContentObserver( - object : ContentObserver(null) { - override fun onChange(selfChange: Boolean, uri: Uri?) { - cursor.close() - val newCursor = query() - if (newCursor == null) { - cont.cancel(IOException("Re-query returned no results")) - } else { - cont.resume(newCursor) - } - } - } - ) - } else { - // not loading, return cursor right away - cont.resume(cursor) - } - } - } -} diff --git a/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/DocumentsStorageProviderIT.kt b/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/DocumentsStorageProviderIT.kt index bf1631108716..3e441bbaeb93 100644 --- a/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/DocumentsStorageProviderIT.kt +++ b/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/DocumentsStorageProviderIT.kt @@ -1,8 +1,8 @@ /* * Nextcloud - Android Client * - * SPDX-FileCopyrightText: 2020 Torsten Grote - * SPDX-License-Identifier: AGPL-3.0-or-later OR GPL-2.0-only + * SPDX-FileCopyrightText: 2026 Alper Ozturk + * SPDX-License-Identifier: AGPL-3.0-or-later */ package com.owncloud.android.providers.documentstorage @@ -19,306 +19,263 @@ import com.owncloud.android.datamodel.UploadsStorageManager import com.owncloud.android.datamodel.UploadsStorageManager.UploadStatus import com.owncloud.android.db.OCUpload import com.owncloud.android.db.UploadResult -import com.owncloud.android.lib.common.utils.Log_OC -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.assertExistsOnServer -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.assertListFilesEquals -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.assertReadEquals -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.assertRecentlyModified -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.assertRegularFile -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.assertRegularFolder -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.findFileBlocking -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.getOCFile -import com.owncloud.android.providers.documentstorage.DocumentsProviderUtils.listFilesBlocking +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.assertContainsAllFiles +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.assertExistsOnServer +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.assertRecentlyModified +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.assertRegularFile +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.assertRegularFolder +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.findChild +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.findOCFile +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.listChildren +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.requireNewDirectory +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.requireNewFile +import com.owncloud.android.providers.documentstorage.TestDocumentsProviderUtils.requireOCFile import com.owncloud.android.providers.documentstorage.DocumentsStorageProvider.Companion.DOCUMENTID_SEPARATOR +import kotlin.random.Random +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource import kotlinx.coroutines.runBlocking import org.apache.commons.httpclient.HttpStatus import org.apache.commons.httpclient.methods.ByteArrayRequestEntity import org.apache.jackrabbit.webdav.client.methods.PutMethod import org.junit.After +import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Assert.fail import org.junit.Before +import org.junit.Rule import org.junit.Test -import kotlin.random.Random - -private const val MAX_FILE_NAME_LENGTH = 225 - -// the provider can only enqueue the upload worker, so give it a generous budget before giving up -private const val UPLOAD_POLL_ATTEMPTS = 45 +import org.junit.rules.Timeout class DocumentsStorageProviderIT : AbstractOnServerIT() { + companion object { + private const val MAX_FILE_NAME_LENGTH = 225 + private const val MIN_RANDOM_KILOBYTES = 1 + private const val MAX_RANDOM_KILOBYTES = 99 + private const val BYTES_PER_KILOBYTE = 1024 + private const val FULL_PERMISSIONS = "RSMCKGWDNV" + private const val WRITE_TRUNCATE_MODE = "wt" + private const val HTML_MIME_TYPE = "text/html" + private const val TEXT_MIME_TYPE = "text/plain" + private const val BINARY_MIME_TYPE = "application/octet-stream" + private const val INITIAL_CONTENT = "initial content" + private const val CHANGED_CONTENT = "new content" + private val TEST_TIMEOUT = 5.minutes + private val AWAIT_TIMEOUT = 90.seconds + private val FOLDER_MODIFICATION_TOLERANCE = 1.minutes + } + private val context = targetContext private val contentResolver = context.contentResolver private val authority = context.getString(R.string.document_provider_authority) private val rootFileId = storageManager.getFileByEncryptedRemotePath(ROOT_PATH).fileId - private val documentId = "${DocumentsStorageProvider.rootIdForUser(user)}${DOCUMENTID_SEPARATOR}$rootFileId" + private val documentId = "${DocumentsStorageProvider.rootIdForUser(user)}$DOCUMENTID_SEPARATOR$rootFileId" private val uri = DocumentsContract.buildTreeDocumentUri(authority, documentId) - private val rootDir get() = DocumentFile.fromTreeUri(context, uri)!! + private val rootDir: DocumentFile + get() = checkNotNull(DocumentFile.fromTreeUri(context, uri)) { "Storage root $uri is not accessible" } private val uploadsStorageManager = UploadsStorageManager( UserAccountManagerImpl.fromContext(context), contentResolver ) + @get:Rule + val timeout: Timeout = Timeout.millis(TEST_TIMEOUT.inWholeMilliseconds) + @Before fun before() { - // an upload left over by a previous test fails once its local file is gone, and WorkManager - // then drops everything appended behind it in the upload chain, including our own uploads - FileUploadHelper.instance().cancel(user.accountName) - - // DocumentsProvider#onCreate() is called when the application is started - // which is *after* AbstractOnServerIT adds the accounts (when the app is freshly installed). - // So we need to query our roots here to ensure that the internal storage map is initialized. - storageManager.run { - val updatedRootPath = getFileByEncryptedRemotePath(ROOT_PATH).apply { - permissions = "RSMCKGWDNV" - } - - saveFile(updatedRootPath) - } + cancelPendingUploads() + grantFullPermissionsOnRoot() + initializeProviderRoots() - contentResolver.query(DocumentsContract.buildRootsUri(authority), null, null, null) assertTrue("Storage root does not exist", rootDir.exists()) assertTrue(rootDir.isDirectory) } - /** - * Delete all files in [rootDir] after each test. - * - * We can't use [AbstractOnServerIT.after] as this is only deleting remote files. - */ @After override fun after() = runBlocking { - rootDir.listFilesBlocking(context).forEach { - Log_OC.e("TEST", "Deleting ${it.name}...") - it.delete() - } + cancelPendingUploads() + rootDir.listChildren(context).forEach { it.delete() } } @Test - fun testCreateDeleteFiles() = runBlocking { - // no files in root initially - assertListFilesEquals(emptyList(), rootDir.listFilesBlocking(context)) + fun testWhenFilesCreatedAndDeletedGivenShouldUpdateListingAndServer() = runBlocking { + assertContainsAllFiles(emptyList(), rootDir.listChildren(context)) - // create first file val name1 = RandomStringGenerator.make() - val type1 = "text/html" - val file1 = rootDir.createFile(type1, name1)!! - - // check assumptions - /* FIXME: mimeType */ - file1.assertRegularFile(name1, 0L, null, rootDir) + val file1 = rootDir.requireNewFile(HTML_MIME_TYPE, name1) + file1.assertRegularFile(name = name1, size = 0L, parent = rootDir) file1.assertRecentlyModified() + assertContainsAllFiles(listOf(file1), rootDir.listChildren(context)) + val remotePath1 = file1.requireOCFile(storageManager).remotePath + assertExistsOnServer(client, remotePath1, true) - // file1 is found in root - assertListFilesEquals(listOf(file1), rootDir.listFilesBlocking(context).toList()) - - // file1 was uploaded - val ocFile1 = file1.getOCFile(storageManager)!! - assertExistsOnServer(client, ocFile1.remotePath, true) - - // create second long file with long file name val name2 = RandomStringGenerator.make(MAX_FILE_NAME_LENGTH) - val type2 = "application/octet-stream" - val file2 = rootDir.createFile(type2, name2)!! - - // file2 was uploaded - val ocFile2 = file2.getOCFile(storageManager)!! - assertExistsOnServer(client, ocFile2.remotePath, true) - - // check assumptions - file2.assertRegularFile(name2, 0L, type2, rootDir) + val file2 = rootDir.requireNewFile(BINARY_MIME_TYPE, name2) + val remotePath2 = file2.requireOCFile(storageManager).remotePath + assertExistsOnServer(client, remotePath2, true) + file2.assertRegularFile(name = name2, size = 0L, mimeType = BINARY_MIME_TYPE, parent = rootDir) file2.assertRecentlyModified() + assertContainsAllFiles(listOf(file1, file2), rootDir.listChildren(context)) - // both files get listed in root - assertListFilesEquals(listOf(file1, file2), rootDir.listFiles().toList()) - - // delete first file assertTrue(file1.delete()) assertFalse(file1.exists()) - assertExistsOnServer(client, ocFile1.remotePath, false) - - // only second file gets listed in root - assertListFilesEquals(listOf(file2), rootDir.listFiles().toList()) + assertExistsOnServer(client, remotePath1, false) + assertContainsAllFiles(listOf(file2), rootDir.listChildren(context)) - // delete also second file assertTrue(file2.delete()) assertFalse(file2.exists()) - assertExistsOnServer(client, ocFile2.remotePath, false) - - // no more files in root - assertListFilesEquals(emptyList(), rootDir.listFilesBlocking(context)) + assertExistsOnServer(client, remotePath2, false) + assertContainsAllFiles(emptyList(), rootDir.listChildren(context)) } @Test - fun testReadWriteFiles() { - // create random file - val file1 = rootDir.createFile("application/octet-stream", RandomStringGenerator.make())!! - file1.assertRegularFile(size = 0L) - - // write random bytes to file - @Suppress("MagicNumber") - val dataSize = Random.nextInt(1, 99) * 1024 - val data1 = Random.nextBytes(dataSize) - contentResolver.openOutputStream(file1.uri, "wt").use { - it!!.write(data1) - } + fun testWhenBytesWrittenGivenShouldReadBackSameBytesAndUpdateSize() { + val file = rootDir.requireNewFile(BINARY_MIME_TYPE, RandomStringGenerator.make()) + file.assertRegularFile(size = 0L) - // read back random bytes - assertReadEquals(data1, contentResolver.openInputStream(file1.uri)) + val data = Random.nextBytes(Random.nextInt(MIN_RANDOM_KILOBYTES, MAX_RANDOM_KILOBYTES) * BYTES_PER_KILOBYTE) + file.writeBytes(data) + awaitStoredLength(file, data.size.toLong()) - // file size was updated correctly - file1.assertRegularFile(size = data1.size.toLong()) + assertArrayEquals(data, file.readBytes()) + file.assertRegularFile(size = data.size.toLong()) } @Test - fun testCreateDeleteFolders() = runBlocking { - // create a new folder - val dirName1 = RandomStringGenerator.make() - val dir1 = rootDir.createDirectory(dirName1)!! - dir1.assertRegularFolder(dirName1, rootDir) - // FIXME about a minute gets lost somewhere after CFO sets the correct time - @Suppress("MagicNumber") - assertTrue(System.currentTimeMillis() - dir1.lastModified() < 60_000) -// dir1.assertRecentlyModified() - - // ensure folder was uploaded to server - val ocDir1 = dir1.getOCFile(storageManager)!! - assertExistsOnServer(client, ocDir1.remotePath, true) - - // create file in folder - val file1 = dir1.createFile("text/html", RandomStringGenerator.make())!! - file1.assertRegularFile(parent = dir1) - val ocFile1 = file1.getOCFile(storageManager)!! - assertExistsOnServer(client, ocFile1.remotePath, true) - - // we find the new file in the created folder and get it in the list - assertEquals(file1.uri.toString(), dir1.findFileBlocking(context, file1.name!!)!!.uri.toString()) - assertListFilesEquals(listOf(file1), dir1.listFilesBlocking(context)) - - // delete folder - dir1.delete() - assertFalse(dir1.exists()) - assertExistsOnServer(client, ocDir1.remotePath, false) - - // ensure file got deleted with it - // since Room was introduced, the file is not automatically updated for some reason. - // however, it is correctly deleted from server, and smoke testing shows it works just fine. - // suspecting a race condition of some sort - // assertFalse(file1.exists()) - assertExistsOnServer(client, ocFile1.remotePath, false) + fun testWhenFolderDeletedGivenShouldRemoveFolderAndChildrenFromServer() = runBlocking { + val folderName = RandomStringGenerator.make() + val folder = rootDir.requireNewDirectory(folderName) + folder.assertRegularFolder(name = folderName, parent = rootDir) + folder.assertRecentlyModified(FOLDER_MODIFICATION_TOLERANCE) + val folderRemotePath = folder.requireOCFile(storageManager).remotePath + assertExistsOnServer(client, folderRemotePath, true) + + val file = folder.requireNewFile(HTML_MIME_TYPE, RandomStringGenerator.make()) + file.assertRegularFile(parent = folder) + val fileRemotePath = file.requireOCFile(storageManager).remotePath + assertExistsOnServer(client, fileRemotePath, true) + + val fileName = checkNotNull(file.name) { "${file.uri} has no name" } + assertEquals(file.uri.toString(), folder.findChild(context, fileName)?.uri.toString()) + assertContainsAllFiles(listOf(file), folder.listChildren(context)) + + assertTrue(folder.delete()) + assertFalse(folder.exists()) + assertExistsOnServer(client, folderRemotePath, false) + assertExistsOnServer(client, fileRemotePath, false) } - @Suppress("MagicNumber") - @Test(timeout = 5 * 60 * 1000) - fun testServerChangedFileContent() { - // create random file - val file1 = rootDir.createFile("text/plain", RandomStringGenerator.make())!! - file1.assertRegularFile(size = 0L) - - val createdETag = file1.getOCFile(storageManager)!!.etagOnServer - - assertTrue(createdETag.isNotEmpty()) + @Test + fun testWhenServerContentChangedGivenShouldReadServerContent() { + val file = createUploadedTextFile() - val content1 = "initial content".toByteArray() + overwriteOnServerWithChangedContent(file.requireOCFile(storageManager).remotePath) - // write content bytes to file - contentResolver.openOutputStream(file1.uri, "wt").use { - it!!.write(content1) - } + assertEquals(CHANGED_CONTENT, file.readText()) + } - awaitUploadedToServer(file1) + @Test + fun testWhenWrittenSuccessivelyGivenShouldReadLatestContent() { + val file = createUploadedTextFile() - val remotePath = file1.getOCFile(storageManager)!!.remotePath + file.writeText(CHANGED_CONTENT) + awaitStoredLength(file, CHANGED_CONTENT.toByteArray().size.toLong()) - val content2 = "new content".toByteArray() + assertEquals(CHANGED_CONTENT, file.readText()) + } - // modify content on server side - val putMethod = PutMethod(client.getFilesDavUri(remotePath)) - putMethod.requestEntity = ByteArrayRequestEntity(content2) - assertEquals(HttpStatus.SC_NO_CONTENT, client.executeMethod(putMethod)) - client.exhaustResponse(putMethod.responseBodyAsStream) - putMethod.releaseConnection() // let the connection available for other methods + private fun cancelPendingUploads() { + FileUploadHelper.instance().cancel(user.accountName) + } - // read back content bytes - val bytes = contentResolver.openInputStream(file1.uri)?.readBytes() ?: ByteArray(0) - assertEquals(String(content2), String(bytes)) + private fun grantFullPermissionsOnRoot() { + val root = storageManager.getFileByEncryptedRemotePath(ROOT_PATH).apply { permissions = FULL_PERMISSIONS } + storageManager.saveFile(root) } - @Test - fun testServerSuccessive() { - // create random file - val file1 = rootDir.createFile("text/plain", RandomStringGenerator.make())!! - file1.assertRegularFile(size = 0L) + private fun initializeProviderRoots() { + contentResolver.query(DocumentsContract.buildRootsUri(authority), null, null, null)?.close() + } - val createdETag = file1.getOCFile(storageManager)!!.etagOnServer + private fun createUploadedTextFile(): DocumentFile { + val file = rootDir.requireNewFile(TEXT_MIME_TYPE, RandomStringGenerator.make()) + file.assertRegularFile(size = 0L) + assertTrue(file.requireOCFile(storageManager).etagOnServer.isNotEmpty()) - assertTrue(createdETag.isNotEmpty()) + file.writeText(INITIAL_CONTENT) + awaitUploadFinished(file) + return file + } - val content1 = "initial content".toByteArray() + private fun overwriteOnServerWithChangedContent(remotePath: String) { + val putMethod = PutMethod(client.getFilesDavUri(remotePath)).apply { + requestEntity = ByteArrayRequestEntity(CHANGED_CONTENT.toByteArray()) + } - // write content bytes to file - contentResolver.openOutputStream(file1.uri, "wt").use { - it!!.write(content1) + try { + assertEquals(HttpStatus.SC_NO_CONTENT, client.executeMethod(putMethod)) + client.exhaustResponse(putMethod.responseBodyAsStream) + } finally { + putMethod.releaseConnection() } + } - awaitUploadedToServer(file1) + private fun DocumentFile.writeBytes(content: ByteArray) { + val outputStream = contentResolver.openOutputStream(uri, WRITE_TRUNCATE_MODE) + checkNotNull(outputStream) { "$uri could not be opened for writing" }.use { it.write(content) } + } - val content2 = "new content".toByteArray() + private fun DocumentFile.writeText(content: String) = writeBytes(content.toByteArray()) - contentResolver.openOutputStream(file1.uri, "wt").use { - it!!.write(content2) - } + private fun DocumentFile.readBytes(): ByteArray { + val inputStream = contentResolver.openInputStream(uri) + return checkNotNull(inputStream) { "$uri could not be opened for reading" }.use { it.readBytes() } + } - awaitWriteSaved(file1, content2.size.toLong()) + private fun DocumentFile.readText(): String = readBytes().decodeToString() - // read back content bytes - val bytes = contentResolver.openInputStream(file1.uri)?.readBytes() ?: ByteArray(0) - assertEquals(String(content2), String(bytes)) + private fun awaitStoredLength(file: DocumentFile, expectedLength: Long) { + awaitUntil({ "write of ${file.uri} was not saved with length $expectedLength" }) { + file.findOCFile(storageManager)?.fileLength == expectedLength + } } - private fun awaitUploadedToServer(file: DocumentFile) { - val remotePath = file.getOCFile(storageManager)!!.remotePath + private fun awaitUploadFinished(file: DocumentFile) { + val remotePath = file.requireOCFile(storageManager).remotePath - repeat(UPLOAD_POLL_ATTEMPTS) { - val upload = upload(remotePath) - if (upload?.uploadStatus == UploadStatus.UPLOAD_FAILED || - (upload?.uploadStatus == UploadStatus.UPLOAD_SUCCEEDED && upload.lastResult != UploadResult.UPLOADED) - ) { + awaitUntil({ "upload of $remotePath did not finish, stored uploads: ${describeUploads()}" }) { + val upload = findUpload(remotePath) + if (upload?.hasFailed() == true) { fail("upload of $remotePath failed with ${upload.lastResult}") } - // the status is stored before the worker deletes the local copy and saves the file, - // so the upload only counts as finished once the worker has released it - if (upload?.uploadStatus == UploadStatus.UPLOAD_SUCCEEDED && - !FileUploadWorker.isUploading(remotePath, user.accountName) - ) { - return - } - - shortSleep() + upload?.uploadStatus == UploadStatus.UPLOAD_SUCCEEDED && !isUploadRunning(remotePath) } - - fail("upload of $remotePath did not finish, stored uploads: ${describeUploads()}") } - private fun awaitWriteSaved(file: DocumentFile, expectedLength: Long) { - repeat(UPLOAD_POLL_ATTEMPTS) { - if (file.getOCFile(storageManager)?.fileLength == expectedLength) { - return - } + private fun isUploadRunning(remotePath: String): Boolean = + FileUploadWorker.isUploading(remotePath, user.accountName) + + private fun OCUpload.hasFailed(): Boolean = uploadStatus == UploadStatus.UPLOAD_FAILED || + (uploadStatus == UploadStatus.UPLOAD_SUCCEEDED && lastResult != UploadResult.UPLOADED) + private fun awaitUntil(failureMessage: () -> String, condition: () -> Boolean) { + val deadline = TimeSource.Monotonic.markNow() + AWAIT_TIMEOUT + while (deadline.hasNotPassedNow()) { + if (condition()) return shortSleep() } - fail("write of ${file.uri} was not saved with length $expectedLength") + if (!condition()) fail(failureMessage()) } - private fun upload(remotePath: String): OCUpload? = uploadsStorageManager + private fun findUpload(remotePath: String): OCUpload? = uploadsStorageManager .getUploadsForAccount(user.accountName) .firstOrNull { it.remotePath == remotePath } diff --git a/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/TestDocumentsProviderUtils.kt b/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/TestDocumentsProviderUtils.kt new file mode 100644 index 000000000000..f0ccc101e196 --- /dev/null +++ b/app/src/androidTest/java/com/owncloud/android/providers/documentstorage/TestDocumentsProviderUtils.kt @@ -0,0 +1,166 @@ +/* + * Nextcloud - Android Client + * + * SPDX-FileCopyrightText: 2026 Alper Ozturk + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +package com.owncloud.android.providers.documentstorage + +import android.content.Context +import android.database.ContentObserver +import android.database.Cursor +import android.net.Uri +import android.provider.DocumentsContract.Document.COLUMN_DOCUMENT_ID +import android.provider.DocumentsContract.Document.COLUMN_MIME_TYPE +import android.provider.DocumentsContract.Document.MIME_TYPE_DIR +import android.provider.DocumentsContract.EXTRA_LOADING +import android.provider.DocumentsContract.buildChildDocumentsUriUsingTree +import android.provider.DocumentsContract.buildDocumentUriUsingTree +import android.provider.DocumentsContract.buildTreeDocumentUri +import android.provider.DocumentsContract.getDocumentId +import androidx.documentfile.provider.DocumentFile +import com.owncloud.android.datamodel.FileDataStorageManager +import com.owncloud.android.datamodel.OCFile +import com.owncloud.android.lib.common.OwnCloudClient +import com.owncloud.android.lib.common.utils.Log_OC +import com.owncloud.android.lib.resources.files.ExistenceCheckRemoteOperation +import com.owncloud.android.providers.documentstorage.DocumentsStorageProvider.Companion.DOCUMENTID_SEPARATOR +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import java.io.IOException +import kotlin.coroutines.resume +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +object TestDocumentsProviderUtils { + + private const val TAG = "DocumentsProviderUtils" + private const val DOCUMENT_ID_COLUMN = 0 + private const val MIME_TYPE_COLUMN = 1 + private val RECENTLY_MODIFIED_TOLERANCE = 15.seconds + private val CURSOR_LOAD_TIMEOUT = 15.seconds + private val CHILD_PROJECTION = arrayOf(COLUMN_DOCUMENT_ID, COLUMN_MIME_TYPE) + + internal fun DocumentFile.findOCFile(storageManager: FileDataStorageManager): OCFile? { + val fileId = getDocumentId(uri).split(DOCUMENTID_SEPARATOR)[1].toLong() + return storageManager.getFileById(fileId) + } + + internal fun DocumentFile.requireOCFile(storageManager: FileDataStorageManager): OCFile = + checkNotNull(findOCFile(storageManager)) { "No stored file for $uri" } + + internal fun DocumentFile.requireNewFile(mimeType: String, displayName: String): DocumentFile = + checkNotNull(createFile(mimeType, displayName)) { "Could not create file $displayName in $uri" } + + internal fun DocumentFile.requireNewDirectory(displayName: String): DocumentFile = + checkNotNull(createDirectory(displayName)) { "Could not create folder $displayName in $uri" } + + internal fun DocumentFile.assertRegularFile( + name: String? = null, + size: Long? = null, + mimeType: String? = null, + parent: DocumentFile? = null + ) { + name?.let { assertEquals(it, this.name) } + assertTrue("$uri does not exist", exists()) + assertTrue(isFile) + assertFalse(isDirectory) + assertFalse(isVirtual) + size?.let { assertEquals(it, length()) } + mimeType?.let { assertEquals(it, type) } + parent?.let { assertEquals(it.uri.toString(), parentFile?.uri.toString()) } + } + + internal fun DocumentFile.assertRegularFolder(name: String? = null, parent: DocumentFile? = null) { + name?.let { assertEquals(it, this.name) } + assertTrue("$uri does not exist", exists()) + assertFalse(isFile) + assertTrue(isDirectory) + assertFalse(isVirtual) + parent?.let { assertEquals(it.uri.toString(), parentFile?.uri.toString()) } + } + + internal fun DocumentFile.assertRecentlyModified(tolerance: Duration = RECENTLY_MODIFIED_TOLERANCE) { + val age = (System.currentTimeMillis() - lastModified()).milliseconds + assertTrue("$name was modified $age ago, expected less than $tolerance", age < tolerance) + } + + internal fun assertExistsOnServer(client: OwnCloudClient, remotePath: String, shouldExist: Boolean) { + val result = ExistenceCheckRemoteOperation(remotePath, !shouldExist).execute(client) + assertTrue("$result", result.isSuccess) + } + + internal fun assertContainsAllFiles(expected: Collection, actual: Collection) { + val expectedUris = expected.map { it.uri.toString() }.toSet() + val actualUris = actual.map { it.uri.toString() }.toSet() + assertTrue("Missing ${expectedUris - actualUris} in $actualUris", actualUris.containsAll(expectedUris)) + (actualUris - expectedUris).forEach { Log_OC.w(TAG, "Found unexpected file: $it") } + } + + internal suspend fun DocumentFile.findChild(context: Context, displayName: String): DocumentFile? = + listChildren(context).firstOrNull { it.name == displayName } + + internal suspend fun DocumentFile.listChildren(context: Context): List = withContext(Dispatchers.IO) { + val childrenUri = buildChildDocumentsUriUsingTree(uri, getDocumentId(uri)) + val cursor = try { + queryUntilLoaded { context.contentResolver.query(childrenUri, CHILD_PROJECTION, null, null, null) } + } catch (e: TimeoutCancellationException) { + throw IOException("Children of $uri did not load", e) + } + + cursor.use { + buildList { + while (it.moveToNext()) { + add(it.toDocumentFile(context, uri)) + } + } + } + } + + private fun Cursor.toDocumentFile(context: Context, parentUri: Uri): DocumentFile { + val documentId = getString(DOCUMENT_ID_COLUMN) + val documentFile = if (getString(MIME_TYPE_COLUMN) == MIME_TYPE_DIR) { + DocumentFile.fromTreeUri(context, buildTreeDocumentUri(parentUri.authority, documentId)) + } else { + DocumentFile.fromSingleUri(context, buildDocumentUriUsingTree(parentUri, documentId)) + } + return checkNotNull(documentFile) { "No document for $documentId" } + } + + private suspend fun queryUntilLoaded(query: () -> Cursor?): Cursor = withTimeout(CURSOR_LOAD_TIMEOUT) { + suspendCancellableCoroutine { continuation -> + val cursor = query() ?: throw IOException("Initial query returned no results") + continuation.invokeOnCancellation { cursor.close() } + + if (!cursor.isLoading()) { + continuation.resume(cursor) + return@suspendCancellableCoroutine + } + + cursor.registerContentObserver( + object : ContentObserver(null) { + override fun onChange(selfChange: Boolean, uri: Uri?) { + if (!continuation.isActive) return + + cursor.close() + val reloadedCursor = query() + if (reloadedCursor == null) { + continuation.cancel(IOException("Re-query returned no results")) + } else { + continuation.resume(reloadedCursor) + } + } + } + ) + } + } + + private fun Cursor.isLoading(): Boolean = extras?.getBoolean(EXTRA_LOADING) == true +} diff --git a/renovate.json5 b/renovate.json5 index a3415a76ebc6..86d0cb4812f9 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -29,11 +29,6 @@ "groupName": "com.google.code.findbugs", "matchCurrentVersion": "= 3.0.1u2", "enabled": false - }, - { - "matchDatasources": ["github-runners"], - "matchFileNames": [".github/workflows/garm.yml"], - "enabled": false } ] } diff --git a/scripts/runCombinedTest.sh b/scripts/runCombinedTest.sh index 86ecf0cc6763..bd3a27f27b9a 100755 --- a/scripts/runCombinedTest.sh +++ b/scripts/runCombinedTest.sh @@ -24,7 +24,7 @@ scripts/deleteOldComments.sh "$BRANCH" "IT" "$DRONE_PULL_REQUEST" scripts/wait_for_emulator.sh || exit 1 -./gradlew installGplayDebugAndroidTest +./gradlew installGplayDebugAndroidTest -Pcoverage gradle_arguments=( -Pcoverage @@ -41,9 +41,7 @@ adb logcat -c adb logcat > logcat.txt & LOGCAT_PID=$! -./gradlew createGplayDebugCoverageReport \ -"${gradle_arguments[@]}" \ --Dorg.gradle.jvmargs="--add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.nio.channels=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED" +./gradlew createGplayDebugCoverageReport "${gradle_arguments[@]}" stat=$? # stop saving logcat