From 833c3d9fd4d07f9b530b34697f22ef52c0dc7096 Mon Sep 17 00:00:00 2001 From: bivashy Date: Sun, 1 Jun 2025 23:56:48 +0500 Subject: [PATCH] Initial implementation of kodik metadata extraction --- .../anyame_backend/api/KodikPlayerAPI.java | 17 ++++ .../api/model/KodikAPIPayload.java | 89 +++++++++++++++++++ .../api/model/KodikEpisode.java | 45 ++++++++++ .../api/model/KodikMetadata.java | 30 +++++++ .../api/model/KodikTranslation.java | 70 +++++++++++++++ .../anyame_backend/config/APIConfig.java | 7 ++ .../controller/ExtractController.java | 57 ++++++++++++ .../exception/KodikAPINotFoundException.java | 10 +++ .../exception/KodikExtractionException.java | 14 +++ .../KodikPlayerNotFoundException.java | 10 +++ .../service/KodikLinkExtrtactService.java | 11 +++ .../KodikMetadataExtractorService.java | 77 ++++++++++++++++ .../service/KodikNetworkService.java | 73 +++++++++++++++ 13 files changed, 510 insertions(+) create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikAPIPayload.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikEpisode.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikMetadata.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikTranslation.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/controller/ExtractController.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikAPINotFoundException.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikExtractionException.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikPlayerNotFoundException.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikLinkExtrtactService.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikMetadataExtractorService.java create mode 100644 src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikNetworkService.java diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/KodikPlayerAPI.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/KodikPlayerAPI.java index 41c08d7..5164737 100644 --- a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/KodikPlayerAPI.java +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/KodikPlayerAPI.java @@ -6,6 +6,7 @@ import retrofit2.http.GET; import retrofit2.http.Query; public interface KodikPlayerAPI { + String DEFAULT_PLAYER_NAME = "Player"; String BASE_PLAYER_URL = "https%3A%2F%2Fkodikdb.com%2Ffind-player%3F"; default String shikimoriURL(String id) { @@ -24,6 +25,22 @@ public interface KodikPlayerAPI { return BASE_PLAYER_URL + "ID%3D" + id; } + default Call getKodikIDPlayer(String id, String token) { + return getPlayer(DEFAULT_PLAYER_NAME, false, kodikIDURL(id), token, id, null, null, null); + } + + default Call getShikimoriPlayer(String id, String token) { + return getPlayer(DEFAULT_PLAYER_NAME, false, shikimoriURL(id), token, null, id, null, null); + } + + default Call getKinopoiskPlayer(String id, String token) { + return getPlayer(DEFAULT_PLAYER_NAME, false, kinopoiskURL(id), token, null, null, id, null); + } + + default Call getIMDBPlayer(String id, String token) { + return getPlayer(DEFAULT_PLAYER_NAME, false, imdbURL(id), token, null, null, null, id); + } + @GET("get-player") Call getPlayer( @Query("title") String title, diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikAPIPayload.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikAPIPayload.java new file mode 100644 index 0000000..edc5fe7 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikAPIPayload.java @@ -0,0 +1,89 @@ +package com.backend.extractor.kodik.service.anyame_backend.api.model; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import okhttp3.FormBody; +import okhttp3.RequestBody; + +/** + * KodikAPIPayload + */ +public class KodikAPIPayload { + private static final Map FIELD_PATTERNS = Map.of( + "d", "var\\s*domain\\s*=\\s*['\"]([^'\"]*)['\"];", + "d_sign", "var\\s*d_sign\\s*=\\s*['\"]([^'\"]*)['\"];", + "pd", "var\\s*pd\\s*=\\s*['\"]([^'\"]*)['\"];", + "pd_sign", "var\\s*pd_sign\\s*=\\s*['\"]([^'\"]*)['\"];", + "ref", "var\\s*ref\\s*=\\s*['\"]([^'\"]*)['\"];", + "ref_sign", "var\\s*ref_sign\\s*=\\s*['\"]([^'\"]*)['\"];", + "type", "videoInfo\\.type\\s*=\\s*['\"]([^'\"]*)['\"];", + "hash", "videoInfo\\.hash\\s*=\\s*['\"]([^'\"]*)['\"];", + "id", "videoInfo\\.id\\s*=\\s*['\"]([^'\"]*)['\"];"); + + private final String document; + private final Map parsedData; + + public KodikAPIPayload(String document) { + this.document = document; + this.parsedData = parseAllFields(); + } + + private Map parseAllFields() { + Map data = new HashMap<>(); + + for (Map.Entry entry : FIELD_PATTERNS.entrySet()) { + String fieldName = entry.getKey(); + String pattern = entry.getValue(); + String value = parsePattern(pattern); + data.put(fieldName, value != null ? value : ""); + } + + data.put("bad_user", "false"); + data.put("info", "{}"); + data.put("cdn_is_working", "true"); + + return data; + } + + private String parsePattern(String pattern) { + Pattern p = Pattern.compile(pattern); + Matcher matcher = p.matcher(document); + return matcher.find() ? matcher.group(1) : null; + } + + public String get(String field) { + return parsedData.get(field); + } + + public Map getAllFields() { + return new HashMap<>(parsedData); + } + + public RequestBody toFormBody() { + FormBody.Builder formBuilder = new FormBody.Builder(); + + for (Map.Entry entry : parsedData.entrySet()) { + formBuilder.add(entry.getKey(), entry.getValue()); + } + + return formBuilder.build(); + } + + @Override + public String toString() { + return "KodikAPIPayload{" + + "d='" + get("d") + '\'' + + ", d_sign='" + get("d_sign") + '\'' + + ", pd='" + get("pd") + '\'' + + ", pd_sign='" + get("pd_sign") + '\'' + + ", ref='" + get("ref") + '\'' + + ", ref_sign='" + get("ref_sign") + '\'' + + ", type='" + get("type") + '\'' + + ", hash='" + get("hash") + '\'' + + ", id='" + get("id") + '\'' + + '}'; + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikEpisode.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikEpisode.java new file mode 100644 index 0000000..822f4c8 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikEpisode.java @@ -0,0 +1,45 @@ +package com.backend.extractor.kodik.service.anyame_backend.api.model; + +/** + * KodikEpisode + */ +public class KodikEpisode { + private final String value; + private final String id; + private final String hash; + private final String title; + private final boolean otherTranslation; + + public KodikEpisode(String value, String id, String hash, String title, boolean otherTranslation) { + this.value = value; + this.id = id; + this.hash = hash; + this.title = title; + this.otherTranslation = otherTranslation; + } + + public String getValue() { + return value; + } + + public String getId() { + return id; + } + + public String getHash() { + return hash; + } + + public String getTitle() { + return title; + } + + public boolean isOtherTranslation() { + return otherTranslation; + } + + @Override + public String toString() { + return title; + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikMetadata.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikMetadata.java new file mode 100644 index 0000000..342a048 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikMetadata.java @@ -0,0 +1,30 @@ +package com.backend.extractor.kodik.service.anyame_backend.api.model; + +import java.util.List; + +/** + * KodikMetadata + */ +public class KodikMetadata { + private final List translations; + private final List episodes; + private final KodikAPIPayload apiPayload; + + public KodikMetadata(List translations, List episodes, KodikAPIPayload apiPayload) { + this.translations = translations; + this.episodes = episodes; + this.apiPayload = apiPayload; + } + + public List getTranslations() { + return translations; + } + + public List getEpisodes() { + return episodes; + } + + public KodikAPIPayload getApiPayload() { + return apiPayload; + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikTranslation.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikTranslation.java new file mode 100644 index 0000000..c879de6 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/api/model/KodikTranslation.java @@ -0,0 +1,70 @@ +package com.backend.extractor.kodik.service.anyame_backend.api.model; + +/** + * KodikTranslation + */ +public class KodikTranslation { + private final String id; + private final String title; + private final String mediaId; + private final String mediaHash; + private final String mediaType; + private final String translationType; + private final int episodeCount; + private final boolean selected; + + public KodikTranslation(String id, String title, String mediaId, String mediaHash, + String mediaType, String translationType, int episodeCount, boolean selected) { + this.id = id; + this.title = title; + this.mediaId = mediaId; + this.mediaHash = mediaHash; + this.mediaType = mediaType; + this.translationType = translationType; + this.episodeCount = episodeCount; + this.selected = selected; + } + + // Getters + public String getId() { + return id; + } + + public String getTitle() { + return title; + } + + public String getMediaId() { + return mediaId; + } + + public String getMediaHash() { + return mediaHash; + } + + public String getMediaType() { + return mediaType; + } + + public String getTranslationType() { + return translationType; + } + + public int getEpisodeCount() { + return episodeCount; + } + + public boolean isSelected() { + return selected; + } + + public String buildUrl(int episodeNumber) { + return String.format("https://kodik.info/%s/%s/%s/720p?min_age=16&first_url=false&season=1&episode=%d", + mediaType, mediaId, mediaHash, episodeNumber); + } + + @Override + public String toString() { + return String.format("%s (%d)", title, episodeCount); + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/config/APIConfig.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/config/APIConfig.java index 206cf9b..c4d3ed9 100644 --- a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/config/APIConfig.java +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/config/APIConfig.java @@ -1,6 +1,9 @@ package com.backend.extractor.kodik.service.anyame_backend.config; import com.backend.extractor.kodik.service.anyame_backend.api.KodikPlayerAPI; + +import okhttp3.OkHttpClient; + import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import retrofit2.Retrofit; @@ -24,4 +27,8 @@ public class APIConfig { return retrofit.create(KodikPlayerAPI.class); } + @Bean + public OkHttpClient httpClient() { + return new OkHttpClient(); + } } diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/controller/ExtractController.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/controller/ExtractController.java new file mode 100644 index 0000000..d68451d --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/controller/ExtractController.java @@ -0,0 +1,57 @@ +package com.backend.extractor.kodik.service.anyame_backend.controller; + +import java.io.IOException; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import com.backend.extractor.kodik.service.anyame_backend.api.KodikPlayerAPI; +import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikMetadata; +import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikPlayerResponse; +import com.backend.extractor.kodik.service.anyame_backend.component.KodikAPITokenProvider; +import com.backend.extractor.kodik.service.anyame_backend.service.KodikMetadataExtractorService; +import com.backend.extractor.kodik.service.anyame_backend.service.KodikNetworkService; + +import retrofit2.Response; + +/** + * ExtractController + */ +@RestController +public class ExtractController { + + private final KodikMetadataExtractorService metadataExtractorService; + private final KodikNetworkService networkService; + private final KodikPlayerAPI kodikPlayerAPI; + private final KodikAPITokenProvider tokenProvider; + + public ExtractController(KodikMetadataExtractorService metadataExtractorService, + KodikNetworkService networkService, + KodikPlayerAPI kodikPlayerAPI, + KodikAPITokenProvider tokenProvider) { + this.metadataExtractorService = metadataExtractorService; + this.networkService = networkService; + this.kodikPlayerAPI = kodikPlayerAPI; + this.tokenProvider = tokenProvider; + } + + @GetMapping("/shikimori") + public KodikMetadata shikimori(@RequestParam("id") String shikimoriId) throws IOException { + Response response = kodikPlayerAPI + .getShikimoriPlayer(shikimoriId, tokenProvider.getKodikToken()).execute(); + if (!response.isSuccessful()) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "cannot find player, check id validity"); + } + KodikPlayerResponse responseResult = response.body(); + return extractMetadata(responseResult); + } + + private KodikMetadata extractMetadata(KodikPlayerResponse response) throws IOException { + String rawLink = response.getLink(); + String rawPage = networkService.fetchPage(rawLink); + return metadataExtractorService.parseMetadata(rawPage); + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikAPINotFoundException.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikAPINotFoundException.java new file mode 100644 index 0000000..4bfa062 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikAPINotFoundException.java @@ -0,0 +1,10 @@ +package com.backend.extractor.kodik.service.anyame_backend.exception; + +/** + * KodikAPINotFoundException + */ +public class KodikAPINotFoundException extends Exception { + public KodikAPINotFoundException(String message) { + super(message); + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikExtractionException.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikExtractionException.java new file mode 100644 index 0000000..dc6c527 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikExtractionException.java @@ -0,0 +1,14 @@ +package com.backend.extractor.kodik.service.anyame_backend.exception; + +/** + * KodikExtractionException + */ +public class KodikExtractionException extends Exception { + public KodikExtractionException(String message, Throwable cause) { + super(message, cause); + } + + public KodikExtractionException(String message) { + super(message); + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikPlayerNotFoundException.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikPlayerNotFoundException.java new file mode 100644 index 0000000..37f7da2 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/exception/KodikPlayerNotFoundException.java @@ -0,0 +1,10 @@ +package com.backend.extractor.kodik.service.anyame_backend.exception; + +/** + * KodikPlayerNotFoundException + */ +public class KodikPlayerNotFoundException extends Exception { + public KodikPlayerNotFoundException(String message) { + super(message); + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikLinkExtrtactService.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikLinkExtrtactService.java new file mode 100644 index 0000000..a4980c0 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikLinkExtrtactService.java @@ -0,0 +1,11 @@ +package com.backend.extractor.kodik.service.anyame_backend.service; + +import org.springframework.stereotype.Service; + +/** + * KodikLinkExtrtactService + */ +@Service +public class KodikLinkExtrtactService { + +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikMetadataExtractorService.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikMetadataExtractorService.java new file mode 100644 index 0000000..7ffab76 --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikMetadataExtractorService.java @@ -0,0 +1,77 @@ +package com.backend.extractor.kodik.service.anyame_backend.service; + +import java.util.ArrayList; +import java.util.List; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Service; + +import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikAPIPayload; +import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikEpisode; +import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikMetadata; +import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikTranslation; + +/** + * KodikMetadataExtractorService + */ +@Service +public class KodikMetadataExtractorService { + public KodikMetadata parseMetadata(String html) { + Document doc = Jsoup.parse(html); + + List translations = parseTranslations(doc); + List episodes = parseEpisodes(doc); + KodikAPIPayload apiPayload = new KodikAPIPayload(html); + + return new KodikMetadata(translations, episodes, apiPayload); + } + + private List parseTranslations(Document doc) { + List translations = new ArrayList<>(); + Elements options = doc.select(".serial-translations-box select option"); + + for (Element option : options) { + String id = option.attr("value"); + String title = option.attr("data-title"); + String mediaId = option.attr("data-media-id"); + String mediaHash = option.attr("data-media-hash"); + String mediaType = option.attr("data-media-type"); + String translationType = option.attr("data-translation-type"); + int episodeCount = parseIntSafely(option.attr("data-episode-count")); + boolean selected = option.hasAttr("selected"); + + translations.add(new KodikTranslation(id, title, mediaId, mediaHash, + mediaType, translationType, episodeCount, selected)); + } + + return translations; + } + + private List parseEpisodes(Document doc) { + List episodes = new ArrayList<>(); + Elements options = doc.select(".serial-series-box select option"); + + for (Element option : options) { + String value = option.attr("value"); + String id = option.attr("data-id"); + String hash = option.attr("data-hash"); + String title = option.attr("data-title"); + boolean otherTranslation = "true".equals(option.attr("data-other-translation")); + + episodes.add(new KodikEpisode(value, id, hash, title, otherTranslation)); + } + + return episodes; + } + + private static int parseIntSafely(String value) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return 0; + } + } +} diff --git a/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikNetworkService.java b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikNetworkService.java new file mode 100644 index 0000000..42b871b --- /dev/null +++ b/src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikNetworkService.java @@ -0,0 +1,73 @@ +package com.backend.extractor.kodik.service.anyame_backend.service; + +import java.io.IOException; +import java.net.URI; +import java.net.URL; + +import org.springframework.stereotype.Service; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** + * KodikNetworkService + */ +@Service +public class KodikNetworkService { + private static final String DEFAULT_PROTOCOL = "https:"; + private final OkHttpClient httpClient; + + public KodikNetworkService(OkHttpClient httpClient) { + this.httpClient = httpClient; + } + + public String fetchPage(String url) throws IOException { + Request request = new Request.Builder() + .url(normalizeUrl(url)) + .build(); + + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Failed to fetch page: " + response); + } + return response.body().string(); + } + } + + public String postFormData(String url, RequestBody body, String referer) throws IOException { + String baseUrl = getBaseUrl(url); + + Request request = new Request.Builder() + .url(url) + .post(body) + .addHeader("Origin", baseUrl) + .addHeader("Referer", referer) + .addHeader("Accept", "application/json, text/javascript, */*; q=0.01") + .build(); + + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("API request failed: " + response); + } + return response.body().string(); + } + } + + private String normalizeUrl(String url) { + if (url.startsWith("//")) { + return DEFAULT_PROTOCOL + url; + } + return url; + } + + private String getBaseUrl(String url) { + try { + URL urlObj = new URI(url).toURL(); + return urlObj.getProtocol() + "://" + urlObj.getHost(); + } catch (Exception e) { + throw new RuntimeException("Invalid URL: " + url, e); + } + } +}