diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b0d5fa..1dcf71f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,40 +12,12 @@ env: jobs: build: runs-on: [self-hosted, Linux, X64, arko, android] - outputs: - ppe-sha: ${{ steps.ppe.outputs.sha }} steps: - uses: actions/checkout@v6 - - name: Get PipePipeExtractor HEAD SHA - id: ppe - run: echo "sha=$(git ls-remote https://github.com/InfinityLoop1308/PipePipeExtractor.git HEAD | cut -f1)" >> "$GITHUB_OUTPUT" - - - uses: actions/cache@v5 - id: ppe-cache - with: - path: ~/.m2/repository/com/github/TeamNewPipe - key: ppe-${{ steps.ppe.outputs.sha }} - - - uses: actions/setup-java@v5 - if: steps.ppe-cache.outputs.cache-hit != 'true' - with: - java-version: "17" - distribution: temurin - - - name: Install PipePipeExtractor to Maven local - if: steps.ppe-cache.outputs.cache-hit != 'true' - working-directory: ${{ runner.temp }} - env: - GRADLE_USER_HOME: ${{ runner.temp }}/gradle-ppe - run: | - git clone --depth 1 https://github.com/InfinityLoop1308/PipePipeExtractor.git PipePipeExtractor - cd PipePipeExtractor - ./gradlew :extractor:publishToMavenLocal :timeago-parser:publishToMavenLocal - - uses: actions/setup-java@v5 with: - java-version: "21" + java-version: "25" distribution: temurin - uses: gradle/actions/setup-gradle@v6 @@ -107,14 +79,9 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 - with: - path: ~/.m2/repository/com/github/TeamNewPipe - key: ppe-${{ needs.build.outputs.ppe-sha }} - - uses: actions/setup-java@v5 with: - java-version: "21" + java-version: "25" distribution: temurin - uses: gradle/actions/setup-gradle@v6 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 3f78dd9..cc8fa9e 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,36 +15,10 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Get PipePipeExtractor HEAD SHA - id: ppe - run: echo "sha=$(git ls-remote https://github.com/InfinityLoop1308/PipePipeExtractor.git HEAD | cut -f1)" >> "$GITHUB_OUTPUT" - - - uses: actions/cache@v5 - id: ppe-cache - with: - path: ~/.m2/repository/com/github/TeamNewPipe - key: ppe-${{ steps.ppe.outputs.sha }} - - - uses: actions/setup-java@v5 - if: steps.ppe-cache.outputs.cache-hit != 'true' - with: - java-version: "17" - distribution: temurin - - - name: Install PipePipeExtractor to Maven local - if: steps.ppe-cache.outputs.cache-hit != 'true' - working-directory: ${{ runner.temp }} - env: - GRADLE_USER_HOME: ${{ runner.temp }}/gradle-ppe - run: | - git clone --depth 1 https://github.com/InfinityLoop1308/PipePipeExtractor.git PipePipeExtractor - cd PipePipeExtractor - ./gradlew :extractor:publishToMavenLocal :timeago-parser:publishToMavenLocal - - - name: Set up JDK 21 + - name: Set up JDK 25 uses: actions/setup-java@v5 with: - java-version: "21" + java-version: "25" distribution: temurin - uses: gradle/actions/setup-gradle@v6 diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index b845c7d..68ea3d3 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v6 - uses: actions/setup-java@v5 with: - java-version: "21" + java-version: "25" distribution: temurin - uses: gradle/actions/setup-gradle@v6 - name: Validate OpenAPI diff --git a/Dockerfile b/Dockerfile index 46cbf4a..1eeee66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,5 @@ -FROM eclipse-temurin:17-jdk-alpine AS ppe-builder -WORKDIR /ppe -RUN apk add --no-cache git && \ - git clone --depth 1 https://github.com/InfinityLoop1308/PipePipeExtractor.git . && \ - GRADLE_USER_HOME=/ppe/.gradle ./gradlew :extractor:publishToMavenLocal :timeago-parser:publishToMavenLocal --no-daemon -q - -FROM eclipse-temurin:21-jdk-alpine AS builder +FROM eclipse-temurin:25-jdk-alpine AS builder WORKDIR /app -COPY --from=ppe-builder /root/.m2 /root/.m2 COPY gradlew ./ COPY gradle/ ./gradle/ RUN ./gradlew --version --no-daemon -q @@ -15,7 +8,7 @@ RUN ./gradlew dependencies --no-daemon -q || true COPY src/ ./src/ RUN ./gradlew shadowJar --no-daemon -q -FROM eclipse-temurin:21-jre-alpine AS runner +FROM eclipse-temurin:25-jre-alpine AS runner RUN addgroup -S typetype && adduser -S typetype -G typetype WORKDIR /app COPY --from=builder /app/build/libs/typetype-server-all.jar app.jar diff --git a/build.gradle.kts b/build.gradle.kts index d4d12b6..85f9e81 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -31,7 +31,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.5.34") - implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:871ea2df92cb81d6bc59967531523b041a9bf462") + implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:290faeb271b589740f13d24a22d775436d395b84") implementation("com.squareup.okhttp3:okhttp:5.3.2") implementation("io.lettuce:lettuce-core:7.6.0.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.3.0") @@ -109,7 +109,7 @@ tasks.check { } kotlin { - jvmToolchain(21) + jvmToolchain(25) sourceSets.named("main") { kotlin.srcDir(generatedBuildInfoDir) } } diff --git a/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt b/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt index 4bc7979..253d3f6 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt @@ -11,7 +11,6 @@ class CachedStreamService( ) : StreamService { companion object { - private const val STREAM_TTL_SECONDS = 21600L fun cacheKey(url: String): String = "stream:$url" } @@ -24,7 +23,8 @@ class CachedStreamService( } val result = delegate.getStreamInfo(url) if (result is ExtractionResult.Success) { - runCatching { cache.set(key, CacheJson.encodeToString(StreamResponse.serializer(), result.data), STREAM_TTL_SECONDS) } + val ttl = result.data.streamCacheTtlSeconds() + if (ttl > 0) runCatching { cache.set(key, CacheJson.encodeToString(StreamResponse.serializer(), result.data), ttl) } } return result } diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt b/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt index f762a12..446fb83 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt +++ b/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt @@ -2,14 +2,17 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.StreamingService import org.schabi.newpipe.extractor.channel.ChannelTabExtractor +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs import org.schabi.newpipe.extractor.search.filter.Filter import org.schabi.newpipe.extractor.search.filter.FilterItem internal fun StreamingService.channelTabExtractor( + url: String, channelId: String, tab: String, sort: String?, ): ChannelTabExtractor { + if (tab == ChannelTabs.SEARCH) return getChannelTabExtractor(channelTabLHFactory.fromUrl(url)) val contentFilter = listOf(FilterItem(Filter.ITEM_IDENTIFIER_UNKNOWN, tab)) val linkHandler = channelTabLHFactory.fromQuery(channelId, contentFilter, sort.toYouTubeChannelTabSortFilter()) return getChannelTabExtractor(linkHandler) diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt index f53f4b1..0333512 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt @@ -46,7 +46,7 @@ class PipePipeChannelService : ChannelService { if (tab != null) { val channelUrl = url.toBaseChannelUrl(tab) val metadata = runCatching { ChannelInfo.getInfo(channelUrl) }.getOrNull() - val extractor = service.channelTabExtractor(channelId(channelUrl, service), tab, sort) + val extractor = service.channelTabExtractor(url, channelId(channelUrl, service), tab, sort) extractor.fetchPage() return ChannelTabInfo.getInfo(extractor).toChannelTabResponse(metadata) } @@ -57,7 +57,7 @@ class PipePipeChannelService : ChannelService { val service = NewPipe.getServiceByUrl(url) val tab = url.toChannelTab(sort) if (tab != null) { - val extractor = service.channelTabExtractor(channelId(url.toBaseChannelUrl(tab), service), tab, sort) + val extractor = service.channelTabExtractor(url, channelId(url.toBaseChannelUrl(tab), service), tab, sort) return extractor.getPage(page).toChannelTabResponse() } return ChannelInfo.getMoreItems(service, url, page).toChannelResponse() @@ -65,6 +65,7 @@ class PipePipeChannelService : ChannelService { private fun String.toChannelTab(sort: String?): String? { if (contains("/shorts", ignoreCase = true)) return ChannelTabs.SHORTS + if (contains("/search", ignoreCase = true)) return ChannelTabs.SEARCH return if (sort != null) ChannelTabs.VIDEOS else null } diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt index 21be002..9baadb0 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt @@ -54,7 +54,7 @@ class PipePipePodcastService : PodcastService { channelUrl: String, page: Page?, ): PodcastPageResponse { - val extractor = service.channelTabExtractor(channelId, ChannelTabs.PODCASTS, null) + val extractor = service.channelTabExtractor(channelUrl, channelId, ChannelTabs.PODCASTS, null) if (page == null) { extractor.fetchPage() return ChannelTabInfo.getInfo(extractor).toPodcastPageResponse(channelUrl) diff --git a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt b/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt new file mode 100644 index 0000000..6a211fc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt @@ -0,0 +1,34 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.StreamResponse + +private const val DEFAULT_STREAM_TTL_SECONDS = 21_600L +private const val BILIBILI_SIGNED_STREAM_MAX_TTL_SECONDS = 3_600L +private const val SIGNED_STREAM_DEADLINE_SAFETY_SECONDS = 300L +private const val MIN_CACHEABLE_STREAM_TTL_SECONDS = 60L + +internal fun StreamResponse.streamCacheTtlSeconds(nowEpochSeconds: Long = System.currentTimeMillis() / 1000): Long { + val deadline = signedMediaUrls().mapNotNull { it.bilibiliDeadline() }.minOrNull() + ?: return DEFAULT_STREAM_TTL_SECONDS + val ttl = deadline - nowEpochSeconds - SIGNED_STREAM_DEADLINE_SAFETY_SECONDS + if (ttl < MIN_CACHEABLE_STREAM_TTL_SECONDS) return 0L + return minOf(ttl, BILIBILI_SIGNED_STREAM_MAX_TTL_SECONDS) +} + +private fun StreamResponse.signedMediaUrls(): Sequence = sequence { + videoStreams.forEach { yield(it.url) } + videoOnlyStreams.forEach { yield(it.url) } + audioStreams.forEach { yield(it.url) } +} + +private fun String.bilibiliDeadline(): Long? { + if (!isBilibiliSignedMediaUrl()) return null + return Regex("""[?&]deadline=(\d+)""").find(this)?.groupValues?.get(1)?.toLongOrNull() + ?: Regex("""[?&]hdnts=exp=(\d+)""").find(this)?.groupValues?.get(1)?.toLongOrNull() +} + +private fun String.isBilibiliSignedMediaUrl(): Boolean = + contains("bilibili", ignoreCase = true) || + contains("bilivideo", ignoreCase = true) || + contains("hdslb.com", ignoreCase = true) || + contains("akamaized", ignoreCase = true) diff --git a/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt index d6f9cb5..61888f8 100644 --- a/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt @@ -13,6 +13,7 @@ import io.ktor.server.routing.routing import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -69,4 +70,14 @@ class ChannelRoutesTest { val response = client.get("/channel?url=https://youtube.com/channel/test") assertEquals(HttpStatusCode.BadRequest, response.status) } + + @Test + fun `GET channel keeps encoded search url`() = withApp { + coEvery { channelService.getChannel(any(), any(), any()) } returns + ExtractionResult.Success(testChannelResponse()) + val response = client.get("/channel?url=https%3A%2F%2Fwww.youtube.com%2F%40test%2Fsearch%3Fquery%3Dtyping") + + assertEquals(HttpStatusCode.OK, response.status) + coVerify { channelService.getChannel("https://www.youtube.com/@test/search?query=typing", null, null) } + } } diff --git a/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt b/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt new file mode 100644 index 0000000..ff7b14e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt @@ -0,0 +1,90 @@ +package dev.typetype.server + +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.models.VideoStreamItem +import dev.typetype.server.services.streamCacheTtlSeconds +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class StreamCacheTtlResolverTest { + + @Test + fun `default stream ttl is six hours`() { + assertEquals(21_600L, response("https://example.com/video.mp4").streamCacheTtlSeconds(nowEpochSeconds = 1_000L)) + } + + @Test + fun `bilibili stream ttl follows signed deadline`() { + val url = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=10000&upsig=x" + assertEquals(1_700L, response(url).streamCacheTtlSeconds(nowEpochSeconds = 8_000L)) + } + + @Test + fun `bilibili stream ttl is capped`() { + val url = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=20000&upsig=x" + assertEquals(3_600L, response(url).streamCacheTtlSeconds(nowEpochSeconds = 8_000L)) + } + + @Test + fun `expired bilibili urls are not cached`() { + val url = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=8100&upsig=x" + assertEquals(0L, response(url).streamCacheTtlSeconds(nowEpochSeconds = 8_000L)) + } + + private fun response(url: String): StreamResponse = StreamResponse( + id = "id", + title = "title", + uploaderName = "uploader", + uploaderUrl = "", + uploaderAvatarUrl = "", + thumbnailUrl = "", + description = "", + duration = 1L, + viewCount = 0L, + likeCount = 0L, + dislikeCount = 0L, + uploadDate = "", + uploaded = -1L, + uploaderSubscriberCount = 0L, + uploaderVerified = false, + category = "", + license = "", + visibility = "", + tags = emptyList(), + streamType = "video_stream", + isShortFormContent = false, + requiresMembership = false, + startPosition = 0L, + streamSegments = emptyList(), + hlsUrl = "", + dashMpdUrl = "", + videoStreams = emptyList(), + audioStreams = emptyList(), + originalAudioTrackId = null, + preferredDefaultAudioTrackId = null, + videoOnlyStreams = listOf(video(url)), + subtitles = emptyList(), + previewFrames = emptyList(), + sponsorBlockSegments = emptyList(), + relatedStreams = emptyList(), + ) + + private fun video(url: String): VideoStreamItem = VideoStreamItem( + url = url, + mimeType = "video/mp4", + format = "MPEG_4", + resolution = "360p", + bitrate = null, + codec = null, + isVideoOnly = true, + itag = -1, + width = 0, + height = 0, + fps = 0, + contentLength = 0L, + initStart = 0L, + initEnd = 0L, + indexStart = 0L, + indexEnd = 0L, + ) +}