diff --git a/.github/workflows/01-build.yml b/.github/workflows/01-build.yml index 2d784e6..0cf1d6c 100644 --- a/.github/workflows/01-build.yml +++ b/.github/workflows/01-build.yml @@ -5,9 +5,9 @@ name: Build library on: push: - branches-ignore: [ master ] + branches-ignore: [ not-master ] pull_request: - branches-ignore: [ master ] + branches-ignore: [ not-master ] jobs: build: @@ -23,4 +23,4 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build with Gradle - run: ./gradlew build + run: ./gradlew build -x test diff --git a/.github/workflows/02-publish.yml b/.github/workflows/02-publish.yml index 429b291..8e417dd 100644 --- a/.github/workflows/02-publish.yml +++ b/.github/workflows/02-publish.yml @@ -4,8 +4,7 @@ name: Publish package on: - push: - branches: [ master ] + workflow_dispatch: {} jobs: build: @@ -24,8 +23,12 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build with Gradle - run: ./gradlew --no-daemon build - - name: Upload to github + run: ./gradlew --no-daemon publish -x test env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./gradlew --no-daemon githubRelease + USERNAME: ${{ secrets.M2_USER }} + PASSWORD: ${{ secrets.M2_PASS }} + REPO: releases +# - name: Upload to github +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# run: ./gradlew --no-daemon githubRelease diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b46e1c --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# Skybot source managers + +> [!IMPORTANT] +> This is compiled with `dev.arbjerg:lavalink` version 2.0.x in order to send artwork urls + +A bunch of extra source managers for LavaPlayer: + +- Mixcloud +- ocremix.org +- Clyp.it +- Reddit +- getyarn.io +- Text To Speech (if prefixed with `speak:`) +- TikTok (in beta, works on _most_ videos) +- PornHub + +## Lavalink users +A lavalink plugin version of these source managers can be found here: https://github.com/DuncteBot/skybot-lavalink-plugin + + +## Installation +Installing this plugin can be done via gradle + +```gradle +repositories { + maven("https://m2.duncte123.dev/releases") +} + +dependencies { + implementation("com.dunctebot:sourcemanagers:VERSION") +} +``` +Replace version with the latest version: ![Maven metadata URL][VERSION] + +## Usage +You are able to register all the source managers via the following code snippet: +```java +AudioPlayerManager playerManager = new DefaultAudioPlayerManager(); // Your lavaplayer player manager +String ttsLange = "en-US"; // The language for the speak/tts item + +DuncteBotSources.registerAll(playerManager, ttsLange); +``` + +[VERSION]: https://img.shields.io/maven-metadata/v?metadataUrl=https%3A%2F%2Fm2.duncte123.dev%2Freleases%2Fcom%2Fdunctebot%2Fsourcemanagers%2Fmaven-metadata.xml diff --git a/build.gradle.kts b/build.gradle.kts index 04349c8..0cd98a0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,15 +21,17 @@ plugins { `java-library` `maven-publish` - id("com.github.breadmoirai.github-release") version "2.2.12" +// id("com.github.breadmoirai.github-release") version "2.2.12" } project.group = "com.dunctebot" -project.version = "1.5.1" +project.version = "1.9.0" val archivesBaseName = "sourcemanagers" repositories { + mavenCentral() jcenter() + maven("https://maven.lavalink.dev/releases") maven { url = uri("https://jitpack.io") @@ -37,27 +39,30 @@ repositories { } dependencies { - // build override for age-restricted videos -// implementation(group = "com.github.duncte123", name = "lavaplayer", version = "be6e364") - implementation(group = "com.sedmelluq", name = "lavaplayer", version = "1.3.67") - implementation(group = "io.sentry", name = "sentry-logback", version = "1.7.17") + compileOnly(group = "dev.arbjerg", name = "lavaplayer", version = "2.0.3") + implementation("org.slf4j:slf4j-api:2.0.7") + implementation("commons-io:commons-io:2.7") + implementation(group = "org.jsoup", name = "jsoup", version = "1.15.3") implementation(group = "com.google.code.findbugs", name = "jsr305", version = "3.0.2") + + testImplementation(group = "dev.arbjerg", name = "lavaplayer", version = "2.0.3") } -configure { +configure { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 } tasks.withType { distributionType = DistributionType.ALL - gradleVersion = "6.8" + gradleVersion = "7.3.3" } val jar: Jar by tasks val build: Task by tasks val clean: Task by tasks +val publish: Task by tasks val sourcesJar = task("sourcesJar") { archiveClassifier.set("sources") @@ -74,8 +79,21 @@ build.apply { } publishing { + repositories { + maven { + name = "duncte123-m2" + url = uri("https://m2.duncte123.dev/releases") + credentials { + username = System.getenv("USERNAME") + password = System.getenv("PASSWORD") + } + authentication { + create("basic") + } + } + } publications { - create("mavenJava") { + register("duncte123-m2") { pom { name.set(archivesBaseName) description.set("Source managers for skybot") @@ -111,7 +129,15 @@ publishing { } } -githubRelease { +publish.apply { + dependsOn(build) + + onlyIf { + System.getenv("USERNAME") != null && System.getenv("PASSWORD") != null + } +} + +/*githubRelease { token(System.getenv("GITHUB_TOKEN")) owner("DuncteBot") repo("skybot-source-managers") @@ -119,4 +145,4 @@ githubRelease { overwrite(false) prerelease(false) body(changelog()) -} +}*/ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 80cf08e..669386b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/jitpack.yml b/jitpack.yml index f4bcb52..d08d908 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -1,2 +1,4 @@ jdk: - openjdk11 +install: + - "./gradlew clean build publish -x test" diff --git a/settings.gradle.kts b/settings.gradle.kts index 8c144c7..14de823 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,2 +1,18 @@ +/* + * Copyright 2021 Duncan "duncte123" Sterken + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + rootProject.name = "skybot-source-managers" diff --git a/src/main/java/com/dunctebot/sourcemanagers/AbstractDuncteBotHttpSource.java b/src/main/java/com/dunctebot/sourcemanagers/AbstractDuncteBotHttpSource.java index f5323f5..767e194 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/AbstractDuncteBotHttpSource.java +++ b/src/main/java/com/dunctebot/sourcemanagers/AbstractDuncteBotHttpSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,26 +16,41 @@ package com.dunctebot.sourcemanagers; -import com.dunctebot.sourcemanagers.pornhub.PornHubAudioSourceManager; import com.sedmelluq.discord.lavaplayer.source.AudioSourceManager; import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; +import com.sedmelluq.discord.lavaplayer.tools.http.HttpContextFilter; import com.sedmelluq.discord.lavaplayer.tools.io.HttpClientTools; import com.sedmelluq.discord.lavaplayer.tools.io.HttpConfigurable; import com.sedmelluq.discord.lavaplayer.tools.io.HttpInterface; import com.sedmelluq.discord.lavaplayer.tools.io.HttpInterfaceManager; +import org.apache.http.HttpResponse; +import org.apache.http.client.CookieStore; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.impl.client.BasicCookieStore; import org.apache.http.impl.client.HttpClientBuilder; import java.util.function.Consumer; import java.util.function.Function; public abstract class AbstractDuncteBotHttpSource implements AudioSourceManager, HttpConfigurable { - private final HttpInterfaceManager httpInterfaceManager; + protected final HttpInterfaceManager httpInterfaceManager; public AbstractDuncteBotHttpSource() { - httpInterfaceManager = HttpClientTools.createDefaultThreadLocalManager(); + this(true); + } + + public AbstractDuncteBotHttpSource(boolean withoutCookies) { + this(HttpClientTools.createDefaultThreadLocalManager(), withoutCookies); + } - httpInterfaceManager.setHttpContextFilter(new PornHubAudioSourceManager.FuckCookies()); + public AbstractDuncteBotHttpSource(HttpInterfaceManager httpInterfaceManager, boolean withoutCookies) { + this.httpInterfaceManager = httpInterfaceManager; + + if (withoutCookies) { + this.httpInterfaceManager.setHttpContextFilter(new FuckCookies()); + } } public HttpInterface getHttpInterface() { @@ -56,4 +71,39 @@ public void configureRequests(Function configurato public void configureBuilder(Consumer configurator) { httpInterfaceManager.configureBuilder(configurator); } + + public static class FuckCookies implements HttpContextFilter { + @Override + public void onContextOpen(HttpClientContext context) { + CookieStore cookieStore = context.getCookieStore(); + + if (cookieStore == null) { + cookieStore = new BasicCookieStore(); + context.setCookieStore(cookieStore); + } + + // Reset cookies for each sequence of requests. + cookieStore.clear(); + } + + @Override + public void onContextClose(HttpClientContext context) { + // Not used + } + + @Override + public void onRequest(HttpClientContext context, HttpUriRequest request, boolean isRepetition) { + // Not used + } + + @Override + public boolean onRequestResponse(HttpClientContext context, HttpUriRequest request, HttpResponse response) { + return false; + } + + @Override + public boolean onRequestException(HttpClientContext context, HttpUriRequest request, Throwable error) { + return false; + } + } } diff --git a/src/main/java/com/dunctebot/sourcemanagers/DuncteBotSources.java b/src/main/java/com/dunctebot/sourcemanagers/DuncteBotSources.java index 0462897..4a997ef 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/DuncteBotSources.java +++ b/src/main/java/com/dunctebot/sourcemanagers/DuncteBotSources.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,30 +17,58 @@ package com.dunctebot.sourcemanagers; import com.dunctebot.sourcemanagers.clypit.ClypitAudioSourceManager; -import com.dunctebot.sourcemanagers.extra.YoutubeContextFilterOverride; import com.dunctebot.sourcemanagers.getyarn.GetyarnAudioSourceManager; +import com.dunctebot.sourcemanagers.mixcloud.MixcloudAudioSourceManager; +import com.dunctebot.sourcemanagers.ocremix.OCRemixAudioSourceManager; import com.dunctebot.sourcemanagers.pornhub.PornHubAudioSourceManager; import com.dunctebot.sourcemanagers.reddit.RedditAudioSourceManager; +import com.dunctebot.sourcemanagers.soundgasm.SoundGasmAudioSourceManager; import com.dunctebot.sourcemanagers.speech.SpeechAudioSourceManager; +import com.dunctebot.sourcemanagers.tiktok.TikTokAudioSourceManager; import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; import com.sedmelluq.discord.lavaplayer.source.youtube.YoutubeAudioSourceManager; public class DuncteBotSources { - public static void registerCustom(AudioPlayerManager playerManager, String speechLanguage, - int playlistPageCount, boolean updateYoutubeData) { + /** + * Registers all custom source managers onto the player manager + * + * @param playerManager Your lavalink player manager + * @param speechLanguage The default language for the TTS engine + */ + public static void registerAll(AudioPlayerManager playerManager, String speechLanguage) { + playerManager.registerSourceManager(new GetyarnAudioSourceManager()); + playerManager.registerSourceManager(new ClypitAudioSourceManager()); + playerManager.registerSourceManager(new SpeechAudioSourceManager(speechLanguage)); + playerManager.registerSourceManager(new PornHubAudioSourceManager()); + playerManager.registerSourceManager(new RedditAudioSourceManager()); + playerManager.registerSourceManager(new OCRemixAudioSourceManager()); + playerManager.registerSourceManager(new TikTokAudioSourceManager()); + playerManager.registerSourceManager(new MixcloudAudioSourceManager()); + playerManager.registerSourceManager(new SoundGasmAudioSourceManager()); + } + /** + * Registers only the sourcemanagers used on DuncteBot, missing sources are as follows + * + *
    + *
  • Mixcloud: they banned my server's ips
    • + *
+ * + * @param playerManager Lavaplayer player manager + * @param speechLanguage Default language for tts + * @param playlistPageCount Overriding the youtube playlist count + */ + public static void registerDuncteBot(AudioPlayerManager playerManager, String speechLanguage, int playlistPageCount) { final YoutubeAudioSourceManager youtubeSource = playerManager.source(YoutubeAudioSourceManager.class); youtubeSource.setPlaylistPageCount(playlistPageCount); - youtubeSource.getMainHttpConfiguration() - .setHttpContextFilter( - YoutubeContextFilterOverride.getOrCreate(updateYoutubeData, youtubeSource.getHttpInterface()) - ); playerManager.registerSourceManager(new GetyarnAudioSourceManager()); playerManager.registerSourceManager(new ClypitAudioSourceManager()); playerManager.registerSourceManager(new SpeechAudioSourceManager(speechLanguage)); playerManager.registerSourceManager(new PornHubAudioSourceManager()); playerManager.registerSourceManager(new RedditAudioSourceManager()); - + playerManager.registerSourceManager(new OCRemixAudioSourceManager()); + playerManager.registerSourceManager(new TikTokAudioSourceManager()); + playerManager.registerSourceManager(new SoundGasmAudioSourceManager()); } } diff --git a/src/main/java/com/dunctebot/sourcemanagers/AudioTrackInfoWithImage.java b/src/main/java/com/dunctebot/sourcemanagers/IWillUseIdentifierInstead.java similarity index 55% rename from src/main/java/com/dunctebot/sourcemanagers/AudioTrackInfoWithImage.java rename to src/main/java/com/dunctebot/sourcemanagers/IWillUseIdentifierInstead.java index 24ce34e..50d9efd 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/AudioTrackInfoWithImage.java +++ b/src/main/java/com/dunctebot/sourcemanagers/IWillUseIdentifierInstead.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,5 @@ package com.dunctebot.sourcemanagers; -import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; - -public class AudioTrackInfoWithImage extends AudioTrackInfo { - - private final String image; - - public AudioTrackInfoWithImage(String title, String author, long length, String identifier, boolean isStream, String uri, String image) { - super(title, author, length, identifier, isStream, uri); - this.image = image; - } - - public String getImage() { - return image; - } +public interface IWillUseIdentifierInstead { } diff --git a/src/main/java/com/dunctebot/sourcemanagers/IdentifiedAudioReference.java b/src/main/java/com/dunctebot/sourcemanagers/IdentifiedAudioReference.java index f03968b..4566e31 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/IdentifiedAudioReference.java +++ b/src/main/java/com/dunctebot/sourcemanagers/IdentifiedAudioReference.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/dunctebot/sourcemanagers/Mp3Track.java b/src/main/java/com/dunctebot/sourcemanagers/Mp3Track.java index 9964cd7..65c4636 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/Mp3Track.java +++ b/src/main/java/com/dunctebot/sourcemanagers/Mp3Track.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,9 +39,13 @@ public Mp3Track(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource manager) { this.manager = manager; } + protected HttpInterface getHttpInterface() { + return this.manager.getHttpInterface(); + } + @Override public void process(LocalAudioTrackExecutor executor) throws Exception { - try (HttpInterface httpInterface = manager.getHttpInterface()) { + try (HttpInterface httpInterface = getHttpInterface()) { loadStream(executor, httpInterface); } } @@ -59,11 +63,15 @@ protected InternalAudioTrack createAudioTrack(AudioTrackInfo trackInfo, Seekable return new Mp3AudioTrack(trackInfo, stream); } + /** + * A special helper to determine the length of the file in milliseconds. + * @return The clip length in milliseconds, for some sources this needs to be set to unknown for them to properly work. + */ protected long getTrackDuration() { return this.trackInfo.length; } - protected String getPlaybackUrl() { + public String getPlaybackUrl() { return this.trackInfo.identifier; } diff --git a/src/main/java/com/dunctebot/sourcemanagers/MpegTrack.java b/src/main/java/com/dunctebot/sourcemanagers/MpegTrack.java index 23ba6eb..8004770 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/MpegTrack.java +++ b/src/main/java/com/dunctebot/sourcemanagers/MpegTrack.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/dunctebot/sourcemanagers/Pair.java b/src/main/java/com/dunctebot/sourcemanagers/Pair.java new file mode 100644 index 0000000..6228970 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/Pair.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022 Duncan "duncte123" Sterken + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers; + +public class Pair { + private final L left; + private final R right; + + public Pair(L left, R right) { + this.left = left; + this.right = right; + } + + public L getLeft() { + return left; + } + + public R getRight() { + return right; + } + + public static Pair of(L left, R right) { + return new Pair<>(left, right); + } + + @Override + public String toString() { + return "Pair{" + + "left=" + left + + ", right=" + right + + '}'; + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/Utils.java b/src/main/java/com/dunctebot/sourcemanagers/Utils.java index 420ecdf..496ca23 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/Utils.java +++ b/src/main/java/com/dunctebot/sourcemanagers/Utils.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,69 @@ package com.dunctebot.sourcemanagers; +import org.apache.http.HttpRequest; + +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.Charset; + public class Utils { + public static final String USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0"; + + public static String urlDecode(String in) { + return URLDecoder.decode(in, Charset.defaultCharset()); + } + + public static String urlEncode(String in) { + return URLEncoder.encode(in, Charset.defaultCharset()); + } + + public static String decryptXor(String input, String key) { + final StringBuilder sb = new StringBuilder(); + + while (key.length() < input.length()) { + key += key; + } + + for (int i = 0; i < input.length(); i += 1) { + final int value1 = input.charAt(i); + final int value2 = key.charAt(i); + + final int xorValue = value1 ^ value2; + + sb.append((char) xorValue); + } + + return sb.toString(); + } public static boolean isURL(String url) { return url.matches("^https?:\\/\\/[-a-zA-Z0-9+&@#\\/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#\\/%=~_|]"); } + public static void fakeChrome(HttpRequest request) { + fakeChrome(request, false); + } + + public static void fakeChrome(HttpRequest request, boolean isVideo) { + request.setHeader("Connection", "keep-alive"); + request.setHeader("DNT", "1"); + request.setHeader("Upgrade-Insecure-Requests", "1"); + request.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,video/mp4,image/avif,image/webp,*/*;q=0.8"); +// request.setHeader("Accept-Encoding", "gzip, deflate, br"); + request.setHeader("Accept-Encoding", "none"); + request.setHeader("TE", "trailers"); + request.setHeader("Accept-Language", "en-US,en;q=0.9"); + + if (isVideo) { + request.setHeader("Sec-Fetch-Dest", "empty"); + } else { + request.setHeader("Sec-Fetch-Dest", "document"); + } + + request.setHeader("Sec-Fetch-Mode", "cors"); + request.setHeader("Sec-Fetch-Site", "same-site"); + request.setHeader("User-Agent", USER_AGENT); + } + } diff --git a/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioSourceManager.java index a65cddc..6ecf556 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioSourceManager.java +++ b/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioSourceManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ import com.sedmelluq.discord.lavaplayer.track.AudioReference; import com.sedmelluq.discord.lavaplayer.track.AudioTrack; import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; +import com.sedmelluq.discord.lavaplayer.track.info.AudioTrackInfoBuilder; import org.apache.commons.io.IOUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -62,11 +63,13 @@ public AudioItem loadItem(AudioPlayerManager manager, AudioReference reference) return AudioReference.NO_TRACK; } - return new IdentifiedAudioReference( + final IdentifiedAudioReference ref = new IdentifiedAudioReference( json.get("Mp3Url").safeText(), reference.identifier, json.get("Title").safeText() ); + + return new ClypitAudioTrack(AudioTrackInfoBuilder.create(ref, null).build(), this); } catch (Exception e) { throw ExceptionTools.wrapUnfriendlyExceptions("Something went wrong", FriendlyException.Severity.SUSPICIOUS, e); diff --git a/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioTrack.java index 3bfe4ac..5e2fd86 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioTrack.java +++ b/src/main/java/com/dunctebot/sourcemanagers/clypit/ClypitAudioTrack.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeContextFilterOverride.java b/src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeContextFilterOverride.java deleted file mode 100644 index 5fea6e0..0000000 --- a/src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeContextFilterOverride.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2020 Duncan "duncte123" Sterken - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dunctebot.sourcemanagers.extra; - -import com.sedmelluq.discord.lavaplayer.source.youtube.YoutubeHttpContextFilter; -import com.sedmelluq.discord.lavaplayer.tools.DataFormatTools; -import com.sedmelluq.discord.lavaplayer.tools.JsonBrowser; -import com.sedmelluq.discord.lavaplayer.tools.io.HttpInterface; -import io.sentry.Sentry; -import org.apache.commons.io.IOUtils; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.protocol.HttpClientContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class YoutubeContextFilterOverride extends YoutubeHttpContextFilter implements Closeable { - private static final Logger logger = LoggerFactory.getLogger(YoutubeContextFilterOverride.class); - private static final Pattern YOUTUBE_INFO_REGEX = Pattern.compile("window\\.ytplayer=\\{\\};\\n?ytcfg\\.set\\((.*)\\);"); - private YoutubeVersionData youtubeVersionData = null; - private final HttpInterface httpInterface; - private final ScheduledExecutorService dataUpdateThread = Executors.newSingleThreadScheduledExecutor((r) -> { - final Thread t = new Thread(r); - t.setName("YouTube-data-updater"); - t.setDaemon(true); - return t; - }); - private static YoutubeContextFilterOverride INSTANCE; - - public synchronized static YoutubeContextFilterOverride getOrCreate(boolean shouldUpdate, HttpInterface httpInterface) { - if (INSTANCE == null) { - INSTANCE = new YoutubeContextFilterOverride(shouldUpdate, httpInterface); - } - - return INSTANCE; - } - - private YoutubeContextFilterOverride(boolean shouldUpdate, HttpInterface httpInterface) { - this.httpInterface = httpInterface; - - if (shouldUpdate) { - dataUpdateThread.scheduleAtFixedRate(this::updateYoutubeData, 0L, 1L, TimeUnit.DAYS); - } - } - - private void updateYoutubeData() { - try { - logger.info("Updating youtube version data"); - this.youtubeVersionData = getYoutubeHeaderDetails(); - logger.info("New youtube version data {}", this.youtubeVersionData); - } catch (IOException e) { - Sentry.capture(e); - logger.error("Failed to capture youtube data", e); - } - } - - private YoutubeVersionData getYoutubeHeaderDetails() throws IOException { - final HttpGet httpGet = new HttpGet("https://www.youtube.com/"); - httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"); - - try (final CloseableHttpResponse response = this.httpInterface.execute(httpGet)) { - final String html = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); - - final Matcher matcher = YOUTUBE_INFO_REGEX.matcher(html); - - if (matcher.find()) { - final String extracted = matcher.group(matcher.groupCount()) - .trim() - .replaceAll("undefined", "null"); - final JsonBrowser json = JsonBrowser.parse(extracted); - - return YoutubeVersionData.fromBrowser(json); - } - - return null; - } - } - - @Override - public void onRequest(HttpClientContext context, HttpUriRequest request, boolean isRepetition) { - super.onRequest(context, request, isRepetition); - - if (youtubeVersionData == null) { - return; - } - - request.setHeader("x-youtube-client-version", youtubeVersionData.getVersion()); - request.setHeader("x-youtube-page-cl", youtubeVersionData.getPageCl()); - request.setHeader("x-youtube-page-label", youtubeVersionData.getLabel()); - request.setHeader("x-youtube-identity-token", youtubeVersionData.getIdToken()); - } - - @Override - public void close() { - dataUpdateThread.shutdown(); - } -} diff --git a/src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeVersionData.java b/src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeVersionData.java deleted file mode 100644 index ef719a6..0000000 --- a/src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeVersionData.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2020 Duncan "duncte123" Sterken - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dunctebot.sourcemanagers.extra; - -import com.sedmelluq.discord.lavaplayer.tools.JsonBrowser; - -public class YoutubeVersionData { - - /* INNERTUBE_CONTEXT_CLIENT_VERSION x-youtube-client-version */ - private final String version; - /* VARIANTS_CHECKSUM x-youtube-variants-checksum */ - //private final String checksum; - /* PAGE_BUILD_LABEL x-youtube-page-label */ - private final String label; - /* ID_TOKEN x-youtube-identity-token */ - private final String idToken; - /* PAGE_CL x-youtube-page-cl */ - private final String pageCl; - /* DEVICE x-youtube-device */ -// private final String device; - - public YoutubeVersionData(String version, String idToken, String label, String pageCl) { - this.version = version; - this.idToken = idToken; - this.label = label; - this.pageCl = pageCl; - } - - public String getVersion() { - return version; - } - - public String getIdToken() { - return idToken; - } - - public String getLabel() { - return label; - } - - public String getPageCl() { - return pageCl; - } - - @Override - public String toString() { - return "YoutubeVersionData{" + - "version='" + version + '\'' + - ", idToken='" + idToken + '\'' + - ", label='" + label + '\'' + - ", pageCl='" + pageCl + '\'' + - '}'; - } - - public static YoutubeVersionData fromBrowser(JsonBrowser json) { - return new YoutubeVersionData( - json.get("INNERTUBE_CONTEXT_CLIENT_VERSION").safeText(), - json.get("ID_TOKEN").safeText(), - json.get("PAGE_BUILD_LABEL").safeText(), - json.get("PAGE_CL").safeText() - ); - } -} diff --git a/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioSourceManager.java index d0b48d6..6e66008 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioSourceManager.java +++ b/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioSourceManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,7 +55,9 @@ public AudioItem loadItem(AudioPlayerManager manager, AudioReference reference) ); return new GetyarnAudioTrack( - AudioTrackInfoBuilder.create(ref, null).build(), + AudioTrackInfoBuilder.create(ref, null) + .setArtworkUrl("https://y.yarn.co/" + videoId + "_screenshot.jpg") + .build(), this ); } diff --git a/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioTrack.java index 2b143e5..d823bbe 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioTrack.java +++ b/src/main/java/com/dunctebot/sourcemanagers/getyarn/GetyarnAudioTrack.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ public AudioTrack makeShallowClone() { } @Override - protected String getPlaybackUrl() { + public String getPlaybackUrl() { return "https://y.yarn.co/" + this.trackInfo.identifier + ".mp4?v=0"; } } diff --git a/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudAudioSourceManager.java new file mode 100644 index 0000000..bfd113d --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudAudioSourceManager.java @@ -0,0 +1,222 @@ +/* + * Copyright 2022 Duncan "duncte123" Sterken & devoxin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers.mixcloud; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; +import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; +import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; +import com.sedmelluq.discord.lavaplayer.tools.JsonBrowser; +import com.sedmelluq.discord.lavaplayer.tools.io.HttpClientTools; +import com.sedmelluq.discord.lavaplayer.track.AudioItem; +import com.sedmelluq.discord.lavaplayer.track.AudioReference; +import com.sedmelluq.discord.lavaplayer.track.AudioTrack; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.dunctebot.sourcemanagers.Utils.urlDecode; + +public class MixcloudAudioSourceManager extends AbstractDuncteBotHttpSource { + private static final String THUMBNAILER_BASE = "https://thumbnailer.mixcloud.com/unsafe/390x390/"; + private static final String GRAPHQL_AUDIO_REQUEST = "query PlayerHeroQuery(\n" + + " $lookup: CloudcastLookup!\n" + + ") {\n" + + " cloudcast: cloudcastLookup(lookup: $lookup) {\n" + + " id\n" + + " name\n" + + " picture {\n" + + " isLight\n" + + " primaryColor\n" + + " darkPrimaryColor: primaryColor(darken: 60)\n" + + " ...UGCImage_picture\n" + + " }\n" + + " owner {\n" + + " ...AudioPageAvatar_user\n" + + " id\n" + + " }\n" + + " restrictedReason\n" + + " seekRestriction\n" + + " ...PlayButton_cloudcast\n" + + " }\n" + + "}\n" + + "\n" + + "fragment AudioPageAvatar_user on User {\n" + + " displayName\n" + + " username\n" + + "}\n" + + "\n" + + "fragment PlayButton_cloudcast on Cloudcast {\n" + + " restrictedReason\n" + + " owner {\n" + + " displayName\n" + + " country\n" + + " username\n" + + " isSubscribedTo\n" + + " isViewer\n" + + " id\n" + + " }\n" + + " slug\n" + + " id\n" + + " isDraft\n" + + " isPlayable\n" + + " streamInfo {\n" + + " hlsUrl\n" + + " dashUrl\n" + + " url\n" + + " uuid\n" + + " }\n" + + " audioLength\n" + + " seekRestriction\n" + + "}\n" + + "\n" + + "fragment UGCImage_picture on Picture {\n" + + " urlRoot\n" + + " primaryColor\n" + + "}\n"; + private static final Pattern URL_REGEX = Pattern.compile("https?://(?:(?:www|beta|m)\\.)?mixcloud\\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)/?"); + + @Override + public String getSourceName() { + return "mixcloud"; + } + + @Override + public AudioItem loadItem(AudioPlayerManager manager, AudioReference reference) { + final Matcher matcher = URL_REGEX.matcher(reference.identifier); + + if (!matcher.matches()) { + return null; + } + + // retry if possible + while (true) { + try { + return this.loadItemOnce(reference, matcher); + } catch (Exception e) { + if (!HttpClientTools.isRetriableNetworkException(e)) { + throw ExceptionTools.wrapUnfriendlyExceptions( + "Loading information for a MixCloud track failed.", + FriendlyException.Severity.FAULT, e); + } + } + } + } + + private AudioItem loadItemOnce(AudioReference reference, Matcher matcher) throws IOException { + final String username = urlDecode(matcher.group(1)); + final String slug = urlDecode(matcher.group(2)); + final JsonBrowser trackInfo = this.extractTrackInfoGraphQl(username, slug); + + if (trackInfo == null) { + return AudioReference.NO_TRACK; + } + + final JsonBrowser restrictedReason = trackInfo.get("restrictedReason"); + + if (!restrictedReason.isNull()) { + throw new FriendlyException( + "Playback of this track is restricted.", + FriendlyException.Severity.COMMON, + new Exception(restrictedReason.text()) + ); + } + + final String picturePath = trackInfo.get("picture").get("urlRoot").text(); + final String title = trackInfo.get("name").text(); + final long duration = trackInfo.get("audioLength").as(Long.class) * 1000; + final String uploader = trackInfo.get("owner").get("username").text(); // displayName + + return new MixcloudAudioTrack( + new AudioTrackInfo( + title, + uploader, + duration, + slug, + false, + reference.identifier, + THUMBNAILER_BASE + picturePath, + null + ), + this + ); + } + + protected JsonBrowser extractTrackInfoGraphQl(String username, String slug) throws IOException { + final var body = JsonBrowser.newMap(); + + body.put("query", GRAPHQL_AUDIO_REQUEST); + + final var variables = JsonBrowser.newMap(); + + variables.put("lookup", new MixcloudLookup( + slug, username + )); + + body.put("variables", variables); + + final HttpPost httpPost = new HttpPost("https://app.mixcloud.com/graphql"); + + httpPost.setEntity(new StringEntity(body.text(), ContentType.APPLICATION_JSON)); + + try (final CloseableHttpResponse res = getHttpInterface().execute(httpPost)) { + final int statusCode = res.getStatusLine().getStatusCode(); + + if (statusCode != 200) { + if (statusCode == 404) { + return null; + } + + throw new IOException("Invalid status code for Mixcloud track page response: " + statusCode); + } + + final String content = IOUtils.toString(res.getEntity().getContent(), StandardCharsets.UTF_8); + final JsonBrowser json = JsonBrowser.parse(content).get("data").get("cloudcast"); + + if (json.get("streamInfo").isNull()) { + return null; + } + + return json; + } + } + + @Override + public boolean isTrackEncodable(AudioTrack track) { + return true; + } + + @Override + public void encodeTrack(AudioTrack track, DataOutput output) { + // nothing to encode + } + + @Override + public AudioTrack decodeTrack(AudioTrackInfo trackInfo, DataInput input) { + return new MixcloudAudioTrack(trackInfo, this); + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudAudioTrack.java new file mode 100644 index 0000000..9383983 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudAudioTrack.java @@ -0,0 +1,75 @@ +/* + * Copyright 2022 Duncan "duncte123" Sterken & devoxin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers.mixcloud; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.dunctebot.sourcemanagers.MpegTrack; +import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; +import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; +import com.sedmelluq.discord.lavaplayer.tools.Units; +import com.sedmelluq.discord.lavaplayer.track.AudioTrack; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; + +import java.io.IOException; +import java.util.Base64; + +import static com.dunctebot.sourcemanagers.Utils.decryptXor; +import static com.dunctebot.sourcemanagers.Utils.urlDecode; + +public class MixcloudAudioTrack extends MpegTrack { + private static final String DECRYPTION_KEY = "IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD"; + + public MixcloudAudioTrack(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource manager) { + super(trackInfo, manager); + } + + @Override + protected long getTrackDuration() { + // supply an unknown content length so lavaplayer actually loads the file + return Units.CONTENT_LENGTH_UNKNOWN; + } + + @Override + public String getPlaybackUrl() { + try { + final var trackInfo = getSourceManager().extractTrackInfoGraphQl( + this.trackInfo.author, + urlDecode(this.trackInfo.identifier) + ); + final String encryptedUrl = trackInfo.get("streamInfo").get("url").text(); + final String xorUrl = new String(Base64.getDecoder().decode(encryptedUrl)); + + return decryptXor(xorUrl, DECRYPTION_KEY); + } catch (IOException e) { + throw ExceptionTools.wrapUnfriendlyExceptions( + "Playback of mixcloud track failed", + FriendlyException.Severity.SUSPICIOUS, + e + ); + } + } + + @Override + protected AudioTrack makeShallowClone() { + return new MixcloudAudioTrack(trackInfo, getSourceManager()); + } + + @Override + public MixcloudAudioSourceManager getSourceManager() { + return (MixcloudAudioSourceManager) super.getSourceManager(); + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudLookup.java b/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudLookup.java new file mode 100644 index 0000000..4485af7 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/mixcloud/MixcloudLookup.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Duncan "duncte123" Sterken + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers.mixcloud; + +public class MixcloudLookup { + private String slug; + private String username; + + public MixcloudLookup(String slug, String username) { + this.slug = slug; + this.username = username; + } + + public String getSlug() { + return slug; + } + + public void setSlug(String slug) { + this.slug = slug; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/ocremix/OCRemixAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/ocremix/OCRemixAudioSourceManager.java new file mode 100644 index 0000000..70c808f --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/ocremix/OCRemixAudioSourceManager.java @@ -0,0 +1,155 @@ +/* + * Skybot, a multipurpose discord bot + * Copyright (C) 2017 Duncan "duncte123" Sterken & Ramid "ramidzkh" Khan & Maurice R S "Sanduhr32" + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.dunctebot.sourcemanagers.ocremix; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; +import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; +import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; +import com.sedmelluq.discord.lavaplayer.track.AudioItem; +import com.sedmelluq.discord.lavaplayer.track.AudioReference; +import com.sedmelluq.discord.lavaplayer.track.AudioTrack; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import javax.annotation.Nullable; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class OCRemixAudioSourceManager extends AbstractDuncteBotHttpSource { + private static final Pattern REMIX_PATTERN = Pattern.compile("(?:https?://(?:www\\.)?ocremix\\.org/remix/)?(?OCR[\\d]+)(?:.*)?"); + + @Override + public String getSourceName() { + return "ocremix"; + } + + @Override + public AudioItem loadItem(AudioPlayerManager manager, AudioReference reference) { + final String identifier = reference.identifier; + final Matcher matcher = REMIX_PATTERN.matcher(identifier); + + // TODO: check if starts with OCR? + if (!matcher.matches()) { + return null; + } + + final String id = matcher.group("id"); + // https://ocremix.org/remix/OCR03310?view=xml + final HttpGet httpGet = new HttpGet("https://ocremix.org/remix/"+id+"?view=xml"); + try (final CloseableHttpResponse response = getHttpInterface().execute(httpGet)) { + final int statusCode = response.getStatusLine().getStatusCode(); + + if (statusCode != 200) { + throw new IOException("Unexpected status code for OCR page response: " + statusCode); + } + + final String xml = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + final Document jsoup = Jsoup.parse(xml); + final OCRTrackMeta meta = this.extractTrackData(jsoup); + + if (meta == null) { + return null; + } + + final AudioTrackInfo info = new AudioTrackInfo( + meta.name, + meta.remixers, + meta.trackLength, + "https://ocremix.org/remix/" + meta.id, + false, + meta.fileName + ); + + return new OCRemixAudioTrack(info, this); + } catch (IOException e) { + throw ExceptionTools.wrapUnfriendlyExceptions("Something went wrong", FriendlyException.Severity.SUSPICIOUS, e); + } + } + + @Override + public boolean isTrackEncodable(AudioTrack track) { + return true; + } + + @Override + public void encodeTrack(AudioTrack track, DataOutput output) { + // nothing to encode + } + + @Override + public AudioTrack decodeTrack(AudioTrackInfo trackInfo, DataInput input) { + return new OCRemixAudioTrack(trackInfo, this); + } + + @Nullable + private OCRTrackMeta extractTrackData(Element elem) { + final Element remix = elem.selectFirst("remix"); + + if (remix == null) { + return null; + } + + final String remixers = elem.selectFirst("remixers") + .children() + .stream() + .map((remixer) -> remixer.attr("name")) + .collect(Collectors.joining(", ")); + + return new OCRTrackMeta( + remix.attr("id"), + remix.attr("name"), + remix.attr("file_name"), + Long.parseLong(remix.attr("track_length")) * 1000, // convert from seconds + remixers + ); + } + + static final class OCRTrackMeta { + public final String id; + public final String name; + public final String fileName; + public final long trackLength; + public final String remixers; + + OCRTrackMeta( + String id, + String name, + String fileName, + long trackLength, // seconds + String remixers + ) { + this.id = id; + this.name = name; + this.fileName = fileName; + this.trackLength = trackLength; + this.remixers = remixers; + } + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/ocremix/OCRemixAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/ocremix/OCRemixAudioTrack.java new file mode 100644 index 0000000..2278975 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/ocremix/OCRemixAudioTrack.java @@ -0,0 +1,63 @@ +/* + * Skybot, a multipurpose discord bot + * Copyright (C) 2017 Duncan "duncte123" Sterken & Ramid "ramidzkh" Khan & Maurice R S "Sanduhr32" + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.dunctebot.sourcemanagers.ocremix; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.dunctebot.sourcemanagers.IWillUseIdentifierInstead; +import com.dunctebot.sourcemanagers.Mp3Track; +import com.sedmelluq.discord.lavaplayer.tools.io.HttpInterface; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; +import com.sedmelluq.discord.lavaplayer.track.playback.LocalAudioTrackExecutor; + +public class OCRemixAudioTrack extends Mp3Track implements IWillUseIdentifierInstead { + private static final String[] MUSIC_HOSTS = { + "iterations.org", + "ocrmirror.org", + "ocr.blueblue.fr", + }; + private int hostIndex = 0; + + public OCRemixAudioTrack(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource manager) { + super(trackInfo, manager); + } + + @Override + public void process(LocalAudioTrackExecutor executor) throws Exception { + // attempt to load all hosts if one fails + try (HttpInterface httpInterface = this.getSourceManager().getHttpInterface()) { + while (this.hostIndex < MUSIC_HOSTS.length) { + try { + loadStream(executor, httpInterface); + break; + } catch (Exception e) { + this.hostIndex++; + + if ((this.hostIndex >= MUSIC_HOSTS.length)) { + throw e; + } + } + } + } + } + + @Override + public String getPlaybackUrl() { + return "https://" + MUSIC_HOSTS[this.hostIndex] + this.trackInfo.uri; + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioSourceManager.java index e950abd..00138e4 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioSourceManager.java +++ b/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioSourceManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,25 +17,18 @@ package com.dunctebot.sourcemanagers.pornhub; import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; -import com.dunctebot.sourcemanagers.AudioTrackInfoWithImage; import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; import com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity; import com.sedmelluq.discord.lavaplayer.tools.JsonBrowser; -import com.sedmelluq.discord.lavaplayer.tools.http.HttpContextFilter; import com.sedmelluq.discord.lavaplayer.track.AudioItem; import com.sedmelluq.discord.lavaplayer.track.AudioReference; import com.sedmelluq.discord.lavaplayer.track.AudioTrack; import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; import org.apache.commons.io.IOUtils; -import org.apache.http.HttpResponse; -import org.apache.http.client.CookieStore; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.impl.client.BasicCookieStore; import java.io.DataInput; import java.io.DataOutput; @@ -45,10 +38,10 @@ import java.util.regex.Pattern; public class PornHubAudioSourceManager extends AbstractDuncteBotHttpSource { - private static final String DOMAIN_PATTERN = "https?://([a-z]+\\.)?pornhub\\.(com|net)"; + private static final String DOMAIN_PATTERN = "https?://([a-z]+\\.)?pornhub\\.(com|net|org)"; public static final Pattern DOMAIN_REGEX = Pattern.compile(DOMAIN_PATTERN); private static final Pattern VIDEO_REGEX = Pattern.compile("^" + DOMAIN_PATTERN + "/view_video\\.php\\?viewkey=([a-zA-Z0-9]+)(?:.*)$"); - private static final Pattern VIDEO_INFO_REGEX = Pattern.compile("var flashvars_\\d+ = (\\{.+})"); + public static final Pattern VIDEO_INFO_REGEX = Pattern.compile("var flashvars_\\d+ = (\\{.+})"); private static final Pattern MODEL_INFO_REGEX = Pattern.compile("var MODEL_PROFILE = (\\{.+})"); @Override @@ -106,7 +99,7 @@ private AudioItem loadItemOnce(AudioReference reference) throws IOException { final String author = modelInfo.get("username").safeText(); final int duration = Integer.parseInt(videoInfo.get("video_duration").safeText()) * 1000; // PornHub returns seconds final Matcher matcher = VIDEO_REGEX.matcher(reference.identifier); - final String identifier = matcher.matches() ? "https://www.pornhub.com/view_video.php?viewkey=" + matcher.group(matcher.groupCount()) : reference.identifier; + final String identifier = matcher.matches() ? matcher.group(matcher.groupCount()) : null; final String uri = reference.identifier; final String imageUrl = videoInfo.get("image_url").safeText(); @@ -120,15 +113,16 @@ private AudioItem loadItemOnce(AudioReference reference) throws IOException { ); } - private AudioTrackInfoWithImage buildInfo(String title, String author, long duration, String identifier, String uri, String imageUrl) { - return new AudioTrackInfoWithImage( + private AudioTrackInfo buildInfo(String title, String author, long duration, String identifier, String uri, String imageUrl) { + return new AudioTrackInfo( title, author, duration, identifier, false, uri, - imageUrl + imageUrl, + null ); } @@ -146,12 +140,7 @@ private PornHubAudioTrack buildAudioTrack(String title, String author, long dura ); } - /*private PornHubAudioTrack buildAudioTrack(AudioTrackInfoWithImage info) { - return new PornHubAudioTrack(info, this); - }*/ - private JsonBrowser getVideoInfo(String html) throws IOException { - // flashvars_130837711['mediaDefinitions'][0]['videoUrl'] final Matcher matcher = VIDEO_INFO_REGEX.matcher(html); if (matcher.find()) { @@ -174,7 +163,7 @@ private JsonBrowser getModelInfo(String html) throws IOException { private String loadHtml(String url) throws IOException { final HttpGet httpGet = new HttpGet(url); - httpGet.setHeader("Cookie", "platform=pc"); + httpGet.setHeader("Cookie", "platform=pc; age_verified=1"); try (final CloseableHttpResponse response = getHttpInterface().execute(httpGet)) { final int statusCode = response.getStatusLine().getStatusCode(); @@ -195,38 +184,7 @@ private void notAvailable() { throw new FriendlyException("This video is not available", Severity.COMMON, null); } - public static class FuckCookies implements HttpContextFilter { - @Override - public void onContextOpen(HttpClientContext context) { - CookieStore cookieStore = context.getCookieStore(); - - if (cookieStore == null) { - cookieStore = new BasicCookieStore(); - context.setCookieStore(cookieStore); - } - - // Reset cookies for each sequence of requests. - cookieStore.clear(); - } - - @Override - public void onContextClose(HttpClientContext context) { - // Not used - } - - @Override - public void onRequest(HttpClientContext context, HttpUriRequest request, boolean isRepetition) { - // Not used - } - - @Override - public boolean onRequestResponse(HttpClientContext context, HttpUriRequest request, HttpResponse response) { - return false; - } - - @Override - public boolean onRequestException(HttpClientContext context, HttpUriRequest request, Throwable error) { - return false; - } + public static String getPlayerPage(String id) { + return "https://www.pornhub.com/view_video.php?viewkey=" + id; } } diff --git a/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioTrack.java index d754a5d..a08fdb6 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioTrack.java +++ b/src/main/java/com/dunctebot/sourcemanagers/pornhub/PornHubAudioTrack.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,24 +19,26 @@ import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; import com.dunctebot.sourcemanagers.MpegTrack; import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; -import com.sedmelluq.discord.lavaplayer.tools.io.HttpInterface; +import com.sedmelluq.discord.lavaplayer.tools.JsonBrowser; import com.sedmelluq.discord.lavaplayer.track.AudioTrack; import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; -import static com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity.SUSPICIOUS; +import static com.dunctebot.sourcemanagers.pornhub.PornHubAudioSourceManager.VIDEO_INFO_REGEX; +import static com.dunctebot.sourcemanagers.pornhub.PornHubAudioSourceManager.getPlayerPage; +import static com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity.*; public class PornHubAudioTrack extends MpegTrack { - private static final Pattern MEDIA_STRING = Pattern.compile("(var\\s+?mediastring.+?)<\\/script>"); private static final Pattern MEDIA_STRING_FILTER = Pattern.compile("\\/\\* \\+ [a-zA-Z0-9_]+ \\+ \\*\\/"); public PornHubAudioTrack(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource sourceManager) { @@ -44,34 +46,78 @@ public PornHubAudioTrack(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource s } @Override - protected String getPlaybackUrl() { + public String getPlaybackUrl() { try { - return loadTrackUrl(this.trackInfo, this.getSourceManager().getHttpInterface()); + return loadFromMediaInfo(); } catch (IOException e) { throw new FriendlyException("Could not load PornHub video", SUSPICIOUS, e); } } - private static String loadTrackUrl(AudioTrackInfo trackInfo, HttpInterface httpInterface) throws IOException { - final HttpGet httpGet = new HttpGet(trackInfo.identifier); + public String loadFromMediaInfo() throws IOException { + final HttpGet httpGet = new HttpGet(getPlayerPage(this.trackInfo.identifier)); - httpGet.setHeader("Cookie", "platform=tv"); + httpGet.setHeader("Cookie", "platform=pc; age_verified=1"); - try (final CloseableHttpResponse response = httpInterface.execute(httpGet)) { + try (final CloseableHttpResponse response = this.getSourceManager().getHttpInterface().execute(httpGet)) { final String html = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); - final Matcher matcher = MEDIA_STRING.matcher(html); - - if (!matcher.find()) { - throw new FriendlyException("Could not find media info", SUSPICIOUS, null); + final Matcher matcher = VIDEO_INFO_REGEX.matcher(html); + + if (matcher.find()) { + final String js = matcher.group(matcher.groupCount()); + final JsonBrowser videoInfo = JsonBrowser.parse(js); + + if (videoInfo.get("video_unavailable_country").asBoolean(false)) { + throw new FriendlyException("Video is not available in your country", COMMON, null); + } + + final JsonBrowser defs = videoInfo.get("mediaDefinitions"); + + if (defs.isNull()) { + throw new FriendlyException("Media info not present", COMMON, null); + } + + int i = 0; + while (!defs.index(i).isNull()) { + // we found the default quality + if ("mp4".equalsIgnoreCase(defs.index(i).get("format").safeText())) { + final String cookies = Arrays.stream(response.getHeaders("Set-Cookie")) + .map(NameValuePair::getValue) + .map((s) -> s.split(";", 2)[0]) + .collect(Collectors.joining("; ")); + final String getMedia = parseJsValueToUrl( + html, + scoupMediaVar(html, "media_" + i) + ); + + return loadMp4Url(getMedia, cookies); + } + + i++; + } + + /*return parseJsValueToUrl( + html, + scoupMediaVar(html, "media_0") // fallback to first item (not mp4) + );*/ } - final String js = matcher.group(matcher.groupCount()); + throw new FriendlyException("Could not find media info", COMMON, null); + } + } + + private String scoupMediaVar(String html, String varName) { + final Pattern pattern = Pattern.compile("(var(?:\\s+)?" + varName + "(?:\\s+)?=(?:\\s+)?[^;]+;)"); + final Matcher matcher = pattern.matcher(html); - return parseJsValueToUrl(html, js); + if (!matcher.find()) { + throw new FriendlyException("Media var has changed, please contact developer", FAULT, null); } + + return matcher.group(matcher.groupCount()); } - private static String parseJsValueToUrl(String htmlPage, String js) { + private String parseJsValueToUrl(String htmlPage, String js) { final String filteredJsValue = MEDIA_STRING_FILTER.matcher(js).replaceAll(""); final String variables = filteredJsValue.split("=")[1].split(";")[0]; final String[] items = variables.split("\\+"); @@ -96,6 +142,38 @@ private static String parseJsValueToUrl(String htmlPage, String js) { return String.join("", videoParts); } + private String loadMp4Url(String jsonPage, String cookie) throws IOException { + final HttpGet mediaGet = new HttpGet(jsonPage); + + mediaGet.setHeader("Cookie", cookie + "; platform=pc; age_verified=1"); + mediaGet.setHeader("Referer", getPlayerPage(this.trackInfo.identifier)); + + try (final CloseableHttpResponse response = this.getSourceManager().getHttpInterface().execute(mediaGet)) { + final String body = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + final JsonBrowser json = JsonBrowser.parse(body); + + for (JsonBrowser info : json.values()) { + if (info.get("defaultQuality").asBoolean(false)) { + return info.get("videoUrl").text(); + } + } + + final JsonBrowser firstItem = json.index(0); + + if (firstItem.isNull()) { + throw new FriendlyException("Video url missing on playback page", FAULT, null); + } + + final String videoUrl = firstItem.get("videoUrl").text(); + + if (videoUrl == null) { + throw new FriendlyException("Video url missing on playback page", FAULT, null); + } + + return videoUrl; + } + } + @Override protected AudioTrack makeShallowClone() { return new PornHubAudioTrack(trackInfo, getSourceManager()); diff --git a/src/main/java/com/dunctebot/sourcemanagers/pornhub/StoredForReference.java b/src/main/java/com/dunctebot/sourcemanagers/pornhub/StoredForReference.java new file mode 100644 index 0000000..e898925 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/pornhub/StoredForReference.java @@ -0,0 +1,233 @@ +/* + * Copyright 2022 Duncan "duncte123" Sterken + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers.pornhub; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; +import com.sedmelluq.discord.lavaplayer.tools.JsonBrowser; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; +import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static com.dunctebot.sourcemanagers.pornhub.PornHubAudioSourceManager.getPlayerPage; +import static com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity.*; + +// stored for reference in case I need them in the future +public class StoredForReference extends PornHubAudioTrack { + + public StoredForReference(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource sourceManager) { + super(trackInfo, sourceManager); + } + + private static final String[] FORMAT_PREFIXES = {"media", "quality", "qualityItems"}; + private static final String FORMAT_REGEX = String.format("(var\\s+(?:%s)_.+)", String.join("|", FORMAT_PREFIXES)); + private static final Pattern FORMAT_PATTERN = Pattern.compile(FORMAT_REGEX); + private static final Pattern MEDIA_STRING = Pattern.compile("(var\\s+?mediastring.+?)<\\/script>"); + private static final Pattern VIDEO_SHOW = Pattern.compile("var\\s+?VIDEO_SHOW\\s+?=\\s+?([^;]+);?<\\/script>"); + + + private String parseQualityItems(String json) throws IOException { + final JsonBrowser parse = JsonBrowser.parse(json); + + if (!parse.isList()) { + return null; + } + + final JsonBrowser url = parse.values().stream() + .filter((it) -> !it.get("url").safeText().isEmpty()) + .findFirst().orElse(null); + + if (url == null) { + return null; + } + + return url.get("url").text(); + } + + private String parseJsValue(String input, Map jsVars) { + String inp = input.replaceAll("/\\*(?:(?!\\*/).)*?\\*/", ""); + + if (input.contains("+")) { + return Arrays.stream(input.split("\\+")) + .map(s -> parseJsValue(s, jsVars)) + .collect(Collectors.joining(" ")); + } + + inp = inp.trim(); + + if (jsVars.containsKey(inp)) { + return jsVars.get(inp); + } + + + // can't remove quotes if less than 2 chars + if (inp.length() < 2) { + return inp; + } + + // remove quotes + if ( + (inp.charAt(0) == '"' && inp.charAt(inp.length() - 1) == '"') || + (inp.charAt(0) == '\'' && inp.charAt(inp.length() - 1) == '\'') + ) { + return inp.substring(1, inp.length() - 1); + } + + return inp; + } + + private String loadTrackUrl_old() throws IOException { + final HttpGet httpGet = new HttpGet(getPlayerPage(this.trackInfo.identifier)); + + httpGet.setHeader("Cookie", "platform=tv; age_verified=1"); + + try (final CloseableHttpResponse response = this.getSourceManager().getHttpInterface().execute(httpGet)) { + final String html = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + final Matcher matcher = MEDIA_STRING.matcher(html); + + if (matcher.find()) { + final String js = matcher.group(matcher.groupCount()); + + return parseJsValueToUrl(html, js); + } + + final Matcher videoMatcher = VIDEO_SHOW.matcher(html); + + if (videoMatcher.find()) { + final String cookies = Arrays.stream(response.getHeaders("Set-Cookie")) + .map(NameValuePair::getValue) + .map((s) -> s.split(";", 2)[0]) + .collect(Collectors.joining("; ")); + final String js = videoMatcher.group(videoMatcher.groupCount()); + + return extractVideoFromVideoShow(js, cookies); + } + + throw new FriendlyException("Could not find media info", SUSPICIOUS, null); + } + } + + private String loadTrackUrl() throws IOException { + final HttpGet httpGet = new HttpGet("https://www.pornhub.com/view_video.php?viewkey=" + this.trackInfo.identifier); + + try (final CloseableHttpResponse response = this.getSourceManager().getHttpInterface().execute(httpGet)) { + final String html = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + + if (Pattern.compile("<[^>]+\\bid=[\"']lockedPlayer").matcher(html).find()) { + throw new FriendlyException("Video " + this.trackInfo.identifier + " is locked.", COMMON, null); + } + + final Map jsVars = extractJsVars(html, FORMAT_PATTERN); + + if (jsVars == null) { + throw new FriendlyException("Could not load js vars", SUSPICIOUS, null); + } + + for (final Map.Entry entry : jsVars.entrySet()) { + if (entry.getKey().startsWith("qualityItems")) { + final String playbackUrl = parseQualityItems(entry.getValue()); + + if (playbackUrl != null) { + return playbackUrl; + } + }/* else if (entry.getKey().startsWith("media") || entry.getKey().startsWith("quality")) { + // TODO: these are probably broken anyway + }*/ + } + + throw new FriendlyException("Could not extract video url", COMMON, null); + } +} + + private Map extractJsVars(String html, Pattern pattern) { + final Matcher matcher = pattern.matcher(html); + + if (!matcher.find()) { + return null; + } + + final String[] assignments = matcher.group(1).split(";"); + final Map jsVars = new HashMap<>(); + + for (String assn : assignments) { + assn = assn.trim(); + + if (assn.isBlank()) { + continue; + } + + assn = assn.replaceFirst("var\\s+", ""); + final String[] parts = assn.split("=", 2); + + jsVars.put(parts[0], this.parseJsValue(parts[1], jsVars)); + } + + return jsVars; + } + + + private String extractVideoFromVideoShow(String obj, String cookie) throws IOException { + final JsonBrowser browser = JsonBrowser.parse(obj); + final String mediaUrl = browser.get("mediaUrl").safeText(); + + System.out.println("https://www.pornhub.com" + mediaUrl); + + final HttpGet mediaGet = new HttpGet("https://www.pornhub.com" + mediaUrl); + + mediaGet.setHeader("Cookie", cookie + "; quality=720; platform=pc; age_verified=1"); + mediaGet.setHeader("Referer", getPlayerPage(this.trackInfo.identifier)); + + try (final CloseableHttpResponse response = this.getSourceManager().getHttpInterface().execute(mediaGet)) { + final String body = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + + System.out.println("body " + body); + + final JsonBrowser json = JsonBrowser.parse(body); + + if (!"OK".equals(json.get("status").safeText())) { + throw new FriendlyException("Pornhub video returned non OK status for video info", COMMON, null); + } + + final String videoUrl = json.get("videoUrl").text(); + + if (videoUrl == null) { + throw new FriendlyException("Video url missing on playback page", FAULT, null); + } + + return videoUrl; + } + } + + + //////// + // Dummy methods + + private String parseJsValueToUrl(String one, String two) { + return null; + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioSourceManager.java index 22034e5..94f2435 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioSourceManager.java +++ b/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioSourceManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ package com.dunctebot.sourcemanagers.reddit; import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; -import com.dunctebot.sourcemanagers.AudioTrackInfoWithImage; import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; @@ -39,10 +38,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.dunctebot.sourcemanagers.Utils.USER_AGENT; import static com.dunctebot.sourcemanagers.Utils.isURL; +import static com.dunctebot.sourcemanagers.reddit.RedditAudioTrack.getPlaybackUrl; import static com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity.COMMON; import static com.sedmelluq.discord.lavaplayer.tools.JsonBrowser.NULL_BROWSER; -import static com.dunctebot.sourcemanagers.reddit.RedditAudioTrack.getPlaybackUrl; public class RedditAudioSourceManager extends AbstractDuncteBotHttpSource { private static final Pattern FULL_LINK_REGEX = Pattern.compile("https:\\/\\/(?:www|old)\\.reddit\\.com\\/r\\/(?:[^\\/]+)\\/(?:[^\\/]+)\\/([^\\/]+)(?:\\/?(?:[^\\/]+)?\\/?)?"); @@ -50,7 +50,7 @@ public class RedditAudioSourceManager extends AbstractDuncteBotHttpSource { public RedditAudioSourceManager() { this.configureBuilder( - (builder) -> builder.setUserAgent("Mozilla/5.0 (compatible; https://github.com/DuncteBot/skybot-source-managers)") + (builder) -> builder.setUserAgent(USER_AGENT) ); } @@ -159,10 +159,10 @@ private boolean canPlayAudio(String id) { } } - private RedditAudioTrack buildTrack(@Nullable JsonBrowser data, String pageURl) { + private AudioItem buildTrack(@Nullable JsonBrowser data, String pageURl) { // If we don't have any data we can return null if (data == null) { - return null; + return AudioReference.NO_TRACK; } final String postHint = data.get("post_hint").safeText(); @@ -177,11 +177,15 @@ private RedditAudioTrack buildTrack(@Nullable JsonBrowser data, String pageURl) final JsonBrowser media = data.get("media").get("reddit_video"); final String url = data.get("url").safeText(); + if (media.get("is_gif").asBoolean(false)) { + throw new FriendlyException("Cannot play gifs", COMMON, null); + } + final Matcher videoLink = VIDEO_LINK_REGEX.matcher(url); // This should never happen unless my regex is wrong if (!videoLink.matches()) { - return null; + return AudioReference.NO_TRACK; } final String videoId = videoLink.group(videoLink.groupCount()); @@ -199,14 +203,15 @@ private RedditAudioTrack buildTrack(@Nullable JsonBrowser data, String pageURl) } return new RedditAudioTrack( - new AudioTrackInfoWithImage( + new AudioTrackInfo( data.get("title").safeText(), "u/" + data.get("author").safeText(), - Long.parseLong(media.get("duration").safeText()) * 1000, + media.get("duration").asLong(1) * 1000, videoId, false, pageURl, - thumbnail + thumbnail, + null ), this ); diff --git a/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioTrack.java index 42503d0..f26bb1d 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioTrack.java +++ b/src/main/java/com/dunctebot/sourcemanagers/reddit/RedditAudioTrack.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; import com.dunctebot.sourcemanagers.MpegTrack; +import com.sedmelluq.discord.lavaplayer.tools.Units; import com.sedmelluq.discord.lavaplayer.track.AudioTrack; import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; @@ -27,12 +28,18 @@ public RedditAudioTrack(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource ma } @Override - protected String getPlaybackUrl() { + public String getPlaybackUrl() { return getPlaybackUrl(this.trackInfo.identifier); } static String getPlaybackUrl(String id) { - return "https://v.redd.it/" + id + "/audio?source=fallback"; + return "https://v.redd.it/" + id + "/DASH_audio.mp4?source=fallback"; + } + + @Override + protected long getTrackDuration() { + // return unknown so we get a more accurate representation of the length + return Units.CONTENT_LENGTH_UNKNOWN; } @Override diff --git a/src/main/java/com/dunctebot/sourcemanagers/soundgasm/SoundGasmAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/soundgasm/SoundGasmAudioSourceManager.java new file mode 100644 index 0000000..5efc1c9 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/soundgasm/SoundGasmAudioSourceManager.java @@ -0,0 +1,135 @@ +/* + * Copyright 2022 Duncan "duncte123" Sterken + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers.soundgasm; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; +import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; +import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; +import com.sedmelluq.discord.lavaplayer.tools.Units; +import com.sedmelluq.discord.lavaplayer.tools.io.HttpClientTools; +import com.sedmelluq.discord.lavaplayer.track.AudioItem; +import com.sedmelluq.discord.lavaplayer.track.AudioReference; +import com.sedmelluq.discord.lavaplayer.track.AudioTrack; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SoundGasmAudioSourceManager extends AbstractDuncteBotHttpSource { + private static final Pattern URL_PATTERN = Pattern.compile("https?:\\/\\/soundgasm\\.net\\/u\\/(?(?[^\\/]+)\\/[^\\/]+)"); + private static final Pattern SOUND_PATTERN = Pattern.compile("m4a:(?:\\s+)?[\"']https:\\/\\/media\\.soundgasm\\.net\\/sounds\\/([^.]+)\\.m4a[\"']"); + private static final Pattern TITLE_PATTERN = Pattern.compile("
([^<]+)<\\/div>"); + + @Override + public String getSourceName() { + return "soundgasm"; + } + + @Override + public AudioItem loadItem(AudioPlayerManager manager, AudioReference reference) { + final String url = reference.identifier; + final Matcher urlMatcher = URL_PATTERN.matcher(url); + + if (!urlMatcher.matches()) { + return null; + } + + final String fetchUrl = "https://soundgasm.net/u/" + urlMatcher.group("path"); + + while (true) { + try { + return loadItemOnce(fetchUrl, urlMatcher); + } catch (Exception e) { + if (!HttpClientTools.isRetriableNetworkException(e)) { + throw ExceptionTools.wrapUnfriendlyExceptions( + "Loading of soundgasm track went wrong", + FriendlyException.Severity.FAULT, + e + ); + } + } + } + } + + private AudioItem loadItemOnce(String fetchUrl, Matcher urlMatcher) throws IOException { + final HttpGet httpGet = new HttpGet(fetchUrl); + + try (final CloseableHttpResponse res = getHttpInterface().execute(httpGet)) { + final int statusCode = res.getStatusLine().getStatusCode(); + + if (statusCode != 200) { + if (statusCode == 404) { + return AudioReference.NO_TRACK; + } + + throw new IOException("Invalid status code for soundgasm track page response: " + statusCode); + } + + final String content = IOUtils.toString(res.getEntity().getContent(), StandardCharsets.UTF_8); + final Matcher soundPatternMatcher = SOUND_PATTERN.matcher(content); + + if (!soundPatternMatcher.find()) { + throw new FriendlyException("Failed to extract audio file", FriendlyException.Severity.FAULT, null); + } + + final Matcher titleMatcher = TITLE_PATTERN.matcher(content); + final String title; + + if (titleMatcher.find()) { + title = titleMatcher.group(titleMatcher.groupCount()); + } else { + title = "Unknown title"; + } + + final String identifier = soundPatternMatcher.group(soundPatternMatcher.groupCount()); + + return new SoundGasmAudioTrack( + new AudioTrackInfo( + title, + urlMatcher.group("author"), + Units.DURATION_MS_UNKNOWN, + identifier, + false, + fetchUrl + ), + this + ); + } + } + + @Override + public boolean isTrackEncodable(AudioTrack track) { + return true; + } + + @Override + public void encodeTrack(AudioTrack track, DataOutput output) { + // nothing to encode + } + + @Override + public AudioTrack decodeTrack(AudioTrackInfo trackInfo, DataInput input) { + return new SoundGasmAudioTrack(trackInfo, this); + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/soundgasm/SoundGasmAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/soundgasm/SoundGasmAudioTrack.java new file mode 100644 index 0000000..1d00c70 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/soundgasm/SoundGasmAudioTrack.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 Duncan "duncte123" Sterken + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers.soundgasm; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.dunctebot.sourcemanagers.MpegTrack; +import com.sedmelluq.discord.lavaplayer.tools.Units; +import com.sedmelluq.discord.lavaplayer.track.AudioTrack; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; + +public class SoundGasmAudioTrack extends MpegTrack { + public SoundGasmAudioTrack(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource manager) { + super(trackInfo, manager); + } + + @Override + protected long getTrackDuration() { + return Units.CONTENT_LENGTH_UNKNOWN; + } + + @Override + public String getPlaybackUrl() { + return "https://media.soundgasm.net/sounds/" + this.trackInfo.identifier + ".m4a"; + } + + @Override + protected AudioTrack makeShallowClone() { + return new SoundGasmAudioTrack(this.trackInfo, getSourceManager()); + } +} diff --git a/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioSourceManager.java index 3b55df1..510f8d3 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioSourceManager.java +++ b/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioSourceManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ package com.dunctebot.sourcemanagers.speech; import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; -import com.dunctebot.sourcemanagers.IdentifiedAudioReference; import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; +import com.sedmelluq.discord.lavaplayer.tools.Units; import com.sedmelluq.discord.lavaplayer.track.AudioItem; import com.sedmelluq.discord.lavaplayer.track.AudioReference; import com.sedmelluq.discord.lavaplayer.track.AudioTrack; @@ -75,7 +75,14 @@ public AudioItem loadItem(AudioPlayerManager manager, AudioReference reference) .replace("%query%", encoded); // Redirect to somewhere else - return new IdentifiedAudioReference(mp3URL, reference.identifier, "Speaking " + data); + return new SpeechAudioTrack(new AudioTrackInfo( + "Speaking " + data, + "TTS B0t", + Units.CONTENT_LENGTH_UNKNOWN, + reference.identifier, + false, + mp3URL + ), this); } @Override diff --git a/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioTrack.java b/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioTrack.java index 96324d1..c44f3d9 100644 --- a/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioTrack.java +++ b/src/main/java/com/dunctebot/sourcemanagers/speech/SpeechAudioTrack.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Duncan "duncte123" Sterken + * Copyright 2021 Duncan "duncte123" Sterken * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,11 +22,15 @@ import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; public class SpeechAudioTrack extends Mp3Track { - SpeechAudioTrack(AudioTrackInfo trackInfo, AbstractDuncteBotHttpSource manager) { super(trackInfo, manager); } + @Override + public String getPlaybackUrl() { + return this.trackInfo.uri; + } + @Override public AudioTrack makeShallowClone() { return new SpeechAudioTrack(trackInfo, getSourceManager()); diff --git a/src/main/java/com/dunctebot/sourcemanagers/tiktok/TikTokAudioSourceManager.java b/src/main/java/com/dunctebot/sourcemanagers/tiktok/TikTokAudioSourceManager.java new file mode 100644 index 0000000..964d6e5 --- /dev/null +++ b/src/main/java/com/dunctebot/sourcemanagers/tiktok/TikTokAudioSourceManager.java @@ -0,0 +1,215 @@ +/* + * Copyright 2021 Duncan "duncte123" Sterken + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dunctebot.sourcemanagers.tiktok; + +import com.dunctebot.sourcemanagers.AbstractDuncteBotHttpSource; +import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager; +import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools; +import com.sedmelluq.discord.lavaplayer.tools.FriendlyException; +import com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity; +import com.sedmelluq.discord.lavaplayer.tools.JsonBrowser; +import com.sedmelluq.discord.lavaplayer.tools.io.HttpInterface; +import com.sedmelluq.discord.lavaplayer.track.AudioItem; +import com.sedmelluq.discord.lavaplayer.track.AudioReference; +import com.sedmelluq.discord.lavaplayer.track.AudioTrack; +import com.sedmelluq.discord.lavaplayer.track.AudioTrackInfo; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.dunctebot.sourcemanagers.Utils.fakeChrome; + +public class TikTokAudioSourceManager extends AbstractDuncteBotHttpSource { + private final TikTokAudioTrackHttpManager httpManager = new TikTokAudioTrackHttpManager(); + private static final String BASE = "https:\\/\\/(?:www\\.|m\\.)?tiktok\\.com"; + private static final String USER = "@(?[^/]+)"; + private static final String VIDEO = "(?