Initial implementation of kodik metadata extraction
This commit is contained in:
@ -6,6 +6,7 @@ import retrofit2.http.GET;
|
||||
import retrofit2.http.Query;
|
||||
|
||||
public interface KodikPlayerAPI {
|
||||
String DEFAULT_PLAYER_NAME = "Player";
|
||||
String BASE_PLAYER_URL = "https%3A%2F%2Fkodikdb.com%2Ffind-player%3F";
|
||||
|
||||
default String shikimoriURL(String id) {
|
||||
@ -24,6 +25,22 @@ public interface KodikPlayerAPI {
|
||||
return BASE_PLAYER_URL + "ID%3D" + id;
|
||||
}
|
||||
|
||||
default Call<KodikPlayerResponse> getKodikIDPlayer(String id, String token) {
|
||||
return getPlayer(DEFAULT_PLAYER_NAME, false, kodikIDURL(id), token, id, null, null, null);
|
||||
}
|
||||
|
||||
default Call<KodikPlayerResponse> getShikimoriPlayer(String id, String token) {
|
||||
return getPlayer(DEFAULT_PLAYER_NAME, false, shikimoriURL(id), token, null, id, null, null);
|
||||
}
|
||||
|
||||
default Call<KodikPlayerResponse> getKinopoiskPlayer(String id, String token) {
|
||||
return getPlayer(DEFAULT_PLAYER_NAME, false, kinopoiskURL(id), token, null, null, id, null);
|
||||
}
|
||||
|
||||
default Call<KodikPlayerResponse> getIMDBPlayer(String id, String token) {
|
||||
return getPlayer(DEFAULT_PLAYER_NAME, false, imdbURL(id), token, null, null, null, id);
|
||||
}
|
||||
|
||||
@GET("get-player")
|
||||
Call<KodikPlayerResponse> getPlayer(
|
||||
@Query("title") String title,
|
||||
|
||||
@ -0,0 +1,89 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.api.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import okhttp3.FormBody;
|
||||
import okhttp3.RequestBody;
|
||||
|
||||
/**
|
||||
* KodikAPIPayload
|
||||
*/
|
||||
public class KodikAPIPayload {
|
||||
private static final Map<String, String> FIELD_PATTERNS = Map.of(
|
||||
"d", "var\\s*domain\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"d_sign", "var\\s*d_sign\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"pd", "var\\s*pd\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"pd_sign", "var\\s*pd_sign\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"ref", "var\\s*ref\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"ref_sign", "var\\s*ref_sign\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"type", "videoInfo\\.type\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"hash", "videoInfo\\.hash\\s*=\\s*['\"]([^'\"]*)['\"];",
|
||||
"id", "videoInfo\\.id\\s*=\\s*['\"]([^'\"]*)['\"];");
|
||||
|
||||
private final String document;
|
||||
private final Map<String, String> parsedData;
|
||||
|
||||
public KodikAPIPayload(String document) {
|
||||
this.document = document;
|
||||
this.parsedData = parseAllFields();
|
||||
}
|
||||
|
||||
private Map<String, String> parseAllFields() {
|
||||
Map<String, String> data = new HashMap<>();
|
||||
|
||||
for (Map.Entry<String, String> entry : FIELD_PATTERNS.entrySet()) {
|
||||
String fieldName = entry.getKey();
|
||||
String pattern = entry.getValue();
|
||||
String value = parsePattern(pattern);
|
||||
data.put(fieldName, value != null ? value : "");
|
||||
}
|
||||
|
||||
data.put("bad_user", "false");
|
||||
data.put("info", "{}");
|
||||
data.put("cdn_is_working", "true");
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private String parsePattern(String pattern) {
|
||||
Pattern p = Pattern.compile(pattern);
|
||||
Matcher matcher = p.matcher(document);
|
||||
return matcher.find() ? matcher.group(1) : null;
|
||||
}
|
||||
|
||||
public String get(String field) {
|
||||
return parsedData.get(field);
|
||||
}
|
||||
|
||||
public Map<String, String> getAllFields() {
|
||||
return new HashMap<>(parsedData);
|
||||
}
|
||||
|
||||
public RequestBody toFormBody() {
|
||||
FormBody.Builder formBuilder = new FormBody.Builder();
|
||||
|
||||
for (Map.Entry<String, String> entry : parsedData.entrySet()) {
|
||||
formBuilder.add(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
return formBuilder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "KodikAPIPayload{" +
|
||||
"d='" + get("d") + '\'' +
|
||||
", d_sign='" + get("d_sign") + '\'' +
|
||||
", pd='" + get("pd") + '\'' +
|
||||
", pd_sign='" + get("pd_sign") + '\'' +
|
||||
", ref='" + get("ref") + '\'' +
|
||||
", ref_sign='" + get("ref_sign") + '\'' +
|
||||
", type='" + get("type") + '\'' +
|
||||
", hash='" + get("hash") + '\'' +
|
||||
", id='" + get("id") + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.api.model;
|
||||
|
||||
/**
|
||||
* KodikEpisode
|
||||
*/
|
||||
public class KodikEpisode {
|
||||
private final String value;
|
||||
private final String id;
|
||||
private final String hash;
|
||||
private final String title;
|
||||
private final boolean otherTranslation;
|
||||
|
||||
public KodikEpisode(String value, String id, String hash, String title, boolean otherTranslation) {
|
||||
this.value = value;
|
||||
this.id = id;
|
||||
this.hash = hash;
|
||||
this.title = title;
|
||||
this.otherTranslation = otherTranslation;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public String getHash() {
|
||||
return hash;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public boolean isOtherTranslation() {
|
||||
return otherTranslation;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return title;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.api.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* KodikMetadata
|
||||
*/
|
||||
public class KodikMetadata {
|
||||
private final List<KodikTranslation> translations;
|
||||
private final List<KodikEpisode> episodes;
|
||||
private final KodikAPIPayload apiPayload;
|
||||
|
||||
public KodikMetadata(List<KodikTranslation> translations, List<KodikEpisode> episodes, KodikAPIPayload apiPayload) {
|
||||
this.translations = translations;
|
||||
this.episodes = episodes;
|
||||
this.apiPayload = apiPayload;
|
||||
}
|
||||
|
||||
public List<KodikTranslation> getTranslations() {
|
||||
return translations;
|
||||
}
|
||||
|
||||
public List<KodikEpisode> getEpisodes() {
|
||||
return episodes;
|
||||
}
|
||||
|
||||
public KodikAPIPayload getApiPayload() {
|
||||
return apiPayload;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,70 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.api.model;
|
||||
|
||||
/**
|
||||
* KodikTranslation
|
||||
*/
|
||||
public class KodikTranslation {
|
||||
private final String id;
|
||||
private final String title;
|
||||
private final String mediaId;
|
||||
private final String mediaHash;
|
||||
private final String mediaType;
|
||||
private final String translationType;
|
||||
private final int episodeCount;
|
||||
private final boolean selected;
|
||||
|
||||
public KodikTranslation(String id, String title, String mediaId, String mediaHash,
|
||||
String mediaType, String translationType, int episodeCount, boolean selected) {
|
||||
this.id = id;
|
||||
this.title = title;
|
||||
this.mediaId = mediaId;
|
||||
this.mediaHash = mediaHash;
|
||||
this.mediaType = mediaType;
|
||||
this.translationType = translationType;
|
||||
this.episodeCount = episodeCount;
|
||||
this.selected = selected;
|
||||
}
|
||||
|
||||
// Getters
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public String getMediaId() {
|
||||
return mediaId;
|
||||
}
|
||||
|
||||
public String getMediaHash() {
|
||||
return mediaHash;
|
||||
}
|
||||
|
||||
public String getMediaType() {
|
||||
return mediaType;
|
||||
}
|
||||
|
||||
public String getTranslationType() {
|
||||
return translationType;
|
||||
}
|
||||
|
||||
public int getEpisodeCount() {
|
||||
return episodeCount;
|
||||
}
|
||||
|
||||
public boolean isSelected() {
|
||||
return selected;
|
||||
}
|
||||
|
||||
public String buildUrl(int episodeNumber) {
|
||||
return String.format("https://kodik.info/%s/%s/%s/720p?min_age=16&first_url=false&season=1&episode=%d",
|
||||
mediaType, mediaId, mediaHash, episodeNumber);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("%s (%d)", title, episodeCount);
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,9 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.config;
|
||||
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.KodikPlayerAPI;
|
||||
|
||||
import okhttp3.OkHttpClient;
|
||||
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import retrofit2.Retrofit;
|
||||
@ -24,4 +27,8 @@ public class APIConfig {
|
||||
return retrofit.create(KodikPlayerAPI.class);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public OkHttpClient httpClient() {
|
||||
return new OkHttpClient();
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,57 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.KodikPlayerAPI;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikMetadata;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikPlayerResponse;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.component.KodikAPITokenProvider;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.service.KodikMetadataExtractorService;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.service.KodikNetworkService;
|
||||
|
||||
import retrofit2.Response;
|
||||
|
||||
/**
|
||||
* ExtractController
|
||||
*/
|
||||
@RestController
|
||||
public class ExtractController {
|
||||
|
||||
private final KodikMetadataExtractorService metadataExtractorService;
|
||||
private final KodikNetworkService networkService;
|
||||
private final KodikPlayerAPI kodikPlayerAPI;
|
||||
private final KodikAPITokenProvider tokenProvider;
|
||||
|
||||
public ExtractController(KodikMetadataExtractorService metadataExtractorService,
|
||||
KodikNetworkService networkService,
|
||||
KodikPlayerAPI kodikPlayerAPI,
|
||||
KodikAPITokenProvider tokenProvider) {
|
||||
this.metadataExtractorService = metadataExtractorService;
|
||||
this.networkService = networkService;
|
||||
this.kodikPlayerAPI = kodikPlayerAPI;
|
||||
this.tokenProvider = tokenProvider;
|
||||
}
|
||||
|
||||
@GetMapping("/shikimori")
|
||||
public KodikMetadata shikimori(@RequestParam("id") String shikimoriId) throws IOException {
|
||||
Response<KodikPlayerResponse> response = kodikPlayerAPI
|
||||
.getShikimoriPlayer(shikimoriId, tokenProvider.getKodikToken()).execute();
|
||||
if (!response.isSuccessful()) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "cannot find player, check id validity");
|
||||
}
|
||||
KodikPlayerResponse responseResult = response.body();
|
||||
return extractMetadata(responseResult);
|
||||
}
|
||||
|
||||
private KodikMetadata extractMetadata(KodikPlayerResponse response) throws IOException {
|
||||
String rawLink = response.getLink();
|
||||
String rawPage = networkService.fetchPage(rawLink);
|
||||
return metadataExtractorService.parseMetadata(rawPage);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.exception;
|
||||
|
||||
/**
|
||||
* KodikAPINotFoundException
|
||||
*/
|
||||
public class KodikAPINotFoundException extends Exception {
|
||||
public KodikAPINotFoundException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.exception;
|
||||
|
||||
/**
|
||||
* KodikExtractionException
|
||||
*/
|
||||
public class KodikExtractionException extends Exception {
|
||||
public KodikExtractionException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public KodikExtractionException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.exception;
|
||||
|
||||
/**
|
||||
* KodikPlayerNotFoundException
|
||||
*/
|
||||
public class KodikPlayerNotFoundException extends Exception {
|
||||
public KodikPlayerNotFoundException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.service;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* KodikLinkExtrtactService
|
||||
*/
|
||||
@Service
|
||||
public class KodikLinkExtrtactService {
|
||||
|
||||
}
|
||||
@ -0,0 +1,77 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikAPIPayload;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikEpisode;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikMetadata;
|
||||
import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikTranslation;
|
||||
|
||||
/**
|
||||
* KodikMetadataExtractorService
|
||||
*/
|
||||
@Service
|
||||
public class KodikMetadataExtractorService {
|
||||
public KodikMetadata parseMetadata(String html) {
|
||||
Document doc = Jsoup.parse(html);
|
||||
|
||||
List<KodikTranslation> translations = parseTranslations(doc);
|
||||
List<KodikEpisode> episodes = parseEpisodes(doc);
|
||||
KodikAPIPayload apiPayload = new KodikAPIPayload(html);
|
||||
|
||||
return new KodikMetadata(translations, episodes, apiPayload);
|
||||
}
|
||||
|
||||
private List<KodikTranslation> parseTranslations(Document doc) {
|
||||
List<KodikTranslation> translations = new ArrayList<>();
|
||||
Elements options = doc.select(".serial-translations-box select option");
|
||||
|
||||
for (Element option : options) {
|
||||
String id = option.attr("value");
|
||||
String title = option.attr("data-title");
|
||||
String mediaId = option.attr("data-media-id");
|
||||
String mediaHash = option.attr("data-media-hash");
|
||||
String mediaType = option.attr("data-media-type");
|
||||
String translationType = option.attr("data-translation-type");
|
||||
int episodeCount = parseIntSafely(option.attr("data-episode-count"));
|
||||
boolean selected = option.hasAttr("selected");
|
||||
|
||||
translations.add(new KodikTranslation(id, title, mediaId, mediaHash,
|
||||
mediaType, translationType, episodeCount, selected));
|
||||
}
|
||||
|
||||
return translations;
|
||||
}
|
||||
|
||||
private List<KodikEpisode> parseEpisodes(Document doc) {
|
||||
List<KodikEpisode> episodes = new ArrayList<>();
|
||||
Elements options = doc.select(".serial-series-box select option");
|
||||
|
||||
for (Element option : options) {
|
||||
String value = option.attr("value");
|
||||
String id = option.attr("data-id");
|
||||
String hash = option.attr("data-hash");
|
||||
String title = option.attr("data-title");
|
||||
boolean otherTranslation = "true".equals(option.attr("data-other-translation"));
|
||||
|
||||
episodes.add(new KodikEpisode(value, id, hash, title, otherTranslation));
|
||||
}
|
||||
|
||||
return episodes;
|
||||
}
|
||||
|
||||
private static int parseIntSafely(String value) {
|
||||
try {
|
||||
return Integer.parseInt(value);
|
||||
} catch (NumberFormatException e) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
package com.backend.extractor.kodik.service.anyame_backend.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.RequestBody;
|
||||
import okhttp3.Response;
|
||||
|
||||
/**
|
||||
* KodikNetworkService
|
||||
*/
|
||||
@Service
|
||||
public class KodikNetworkService {
|
||||
private static final String DEFAULT_PROTOCOL = "https:";
|
||||
private final OkHttpClient httpClient;
|
||||
|
||||
public KodikNetworkService(OkHttpClient httpClient) {
|
||||
this.httpClient = httpClient;
|
||||
}
|
||||
|
||||
public String fetchPage(String url) throws IOException {
|
||||
Request request = new Request.Builder()
|
||||
.url(normalizeUrl(url))
|
||||
.build();
|
||||
|
||||
try (Response response = httpClient.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
throw new IOException("Failed to fetch page: " + response);
|
||||
}
|
||||
return response.body().string();
|
||||
}
|
||||
}
|
||||
|
||||
public String postFormData(String url, RequestBody body, String referer) throws IOException {
|
||||
String baseUrl = getBaseUrl(url);
|
||||
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.post(body)
|
||||
.addHeader("Origin", baseUrl)
|
||||
.addHeader("Referer", referer)
|
||||
.addHeader("Accept", "application/json, text/javascript, */*; q=0.01")
|
||||
.build();
|
||||
|
||||
try (Response response = httpClient.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
throw new IOException("API request failed: " + response);
|
||||
}
|
||||
return response.body().string();
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeUrl(String url) {
|
||||
if (url.startsWith("//")) {
|
||||
return DEFAULT_PROTOCOL + url;
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
private String getBaseUrl(String url) {
|
||||
try {
|
||||
URL urlObj = new URI(url).toURL();
|
||||
return urlObj.getProtocol() + "://" + urlObj.getHost();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Invalid URL: " + url, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user