From 04693f99e7b56514a5e17f51720b84946abc3989 Mon Sep 17 00:00:00 2001 From: bivashy Date: Sun, 4 Jan 2026 05:10:15 +0500 Subject: [PATCH 1/5] Initial implementation of `yt-dlp` integration --- Dockerfile.dev | 13 + pom.xml | 26 ++ .../composer/controller/TrackController.java | 39 +-- .../importing/ProgressSSEController.java | 19 +- .../dto/importing/BaseTrackProgress.java | 50 ++++ .../dto/importing/PlaylistProgress.java | 37 +++ .../dto/importing/ProgressEntryType.java | 7 + .../dto/importing/SingleTrackProgress.java | 21 ++ .../dto/importing/TrackProgressDTO.java | 116 --------- .../dto/track/AddYoutubeTrackRequest.java | 4 + .../track/service/AddLocalTrackParams.java | 9 + .../exception/ImportTrackException.java | 11 + .../service/AudioBlobStorageService.java | 4 + .../service/AudioS3StorageService.java | 44 +++- .../composer/service/TrackService.java | 242 +++++++++++++++++- .../composer/service/TrackSourceService.java | 23 +- .../importing/RedisProgressService.java | 18 +- .../backend/composer/util/SimpleBlob.java | 64 +++++ 18 files changed, 577 insertions(+), 170 deletions(-) create mode 100644 src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java create mode 100644 src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java create mode 100644 src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java create mode 100644 src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java delete mode 100644 src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java create mode 100644 src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java create mode 100644 src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java create mode 100644 src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java create mode 100644 src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java diff --git a/Dockerfile.dev b/Dockerfile.dev index 220f6bc..d6f4884 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -5,10 +5,23 @@ RUN apt-get update && \ curl \ vim \ git \ + unzip \ + python3.11 \ ca-certificates \ ffmpeg && \ rm -rf /var/lib/apt/lists/* +RUN ln -sf /usr/bin/python3.11 /usr/bin/python3 + +RUN curl -fsSL https://bun.sh/install > install.sh && \ + chmod +x install.sh && \ + ./install.sh && \ + rm install.sh + +RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \ + chmod a+rx /usr/local/bin/yt-dlp + + # Create non-root user RUN groupadd --gid 1000 spring-app && \ useradd --uid 1000 --gid spring-app --shell /bin/bash --create-home spring-app diff --git a/pom.xml b/pom.xml index 047fca2..ae9b992 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,8 @@ 3.2.3 2.8.5 2024.08.29 + 2.0.6 + 51 @@ -124,6 +126,17 @@ jaffree ${jaffree.version} + + io.github.bivashy + yt-dlp-java + ${yt-dlp-java.version} + + + io.soabase.record-builder + record-builder-core + ${record-builder.version} + provided + org.postgresql @@ -160,6 +173,19 @@ org.springframework.boot spring-boot-maven-plugin + + org.apache.maven.plugins + maven-compiler-plugin + + + + io.soabase.record-builder + record-builder-processor + ${record-builder.version} + + + + diff --git a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java index 04508b9..c249ebb 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java @@ -1,6 +1,5 @@ package com.bivashy.backend.composer.controller; -import java.io.IOException; import java.util.List; import org.springframework.http.MediaType; @@ -14,40 +13,42 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; import com.bivashy.backend.composer.auth.CustomUserDetails; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; import com.bivashy.backend.composer.dto.track.AddLocalTrackRequest; +import com.bivashy.backend.composer.dto.track.AddYoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; -import com.bivashy.backend.composer.model.SourceTypes; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; import com.bivashy.backend.composer.service.TrackService; -import com.bivashy.backend.composer.service.importing.RedisProgressService; +import com.bivashy.backend.composer.util.SimpleBlob.MultipartBlob; @RestController public class TrackController { private final TrackService trackService; - private final RedisProgressService redisProgressService; - public TrackController(TrackService trackService, RedisProgressService redisProgressService) { + public TrackController(TrackService trackService) { this.trackService = trackService; - this.redisProgressService = redisProgressService; } @PostMapping(path = "/playlist/{playlistId}/track/local", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResponseEntity addLocalTrack( @AuthenticationPrincipal CustomUserDetails user, - @PathVariable Long playlistId, - @ModelAttribute AddLocalTrackRequest request) throws IOException { - TrackResponse response = trackService.addLocalTrack(user, playlistId, request); - redisProgressService.saveProgress(new TrackProgressDTO(playlistId, - response.trackId(), - response.title(), - response.fileFormat(), - SourceTypes.FILE, - 100, - null, - System.currentTimeMillis(), - user.getId())); + @PathVariable long playlistId, + @ModelAttribute AddLocalTrackRequest request) throws ImportTrackException { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new MultipartBlob(request.source())) + .build(); + TrackResponse response = trackService.addLocalTrack(user, playlistId, params); + return ResponseEntity.ok(response); + } + + @PostMapping(path = "/playlist/{playlistId}/track/youtube") + public ResponseEntity> addYoutubeTrack( + @AuthenticationPrincipal CustomUserDetails user, + @PathVariable long playlistId, + @RequestBody AddYoutubeTrackRequest request) throws ImportTrackException { + List response = trackService.addYoutubeTrack(user, playlistId, request); return ResponseEntity.ok(response); } diff --git a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java index 09838be..dfdec00 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java @@ -1,13 +1,19 @@ package com.bivashy.backend.composer.controller.importing; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + import org.springframework.http.MediaType; import org.springframework.http.codec.ServerSentEvent; import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; import com.bivashy.backend.composer.auth.CustomUserDetails; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; import com.bivashy.backend.composer.dto.importing.ImportTrackKey; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; import com.bivashy.backend.composer.service.importing.RedisMessageSubscriber; import com.bivashy.backend.composer.service.importing.RedisProgressService; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,10 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.http.HttpServletResponse; import reactor.core.publisher.Flux; import reactor.core.publisher.Sinks; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; @RestController public class ProgressSSEController { @@ -65,11 +67,12 @@ public class ProgressSSEController { .build()) .doFirst(() -> { try { - List existingProgresses = redisProgressService.getPlaylistProgress(playlistId, + List existingProgresses = redisProgressService.getPlaylistProgress( + playlistId, userId); ObjectMapper mapper = new ObjectMapper(); - for (TrackProgressDTO progress : existingProgresses) { + for (BaseTrackProgress progress : existingProgresses) { sink.tryEmitNext(mapper.writeValueAsString(progress)); } } catch (Exception e) { diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java new file mode 100644 index 0000000..da33855 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java @@ -0,0 +1,50 @@ +package com.bivashy.backend.composer.dto.importing; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = PlaylistProgress.class, name = "PLAYLIST"), + @JsonSubTypes.Type(value = SingleTrackProgress.class, name = "TRACK"), +}) +public abstract class BaseTrackProgress { + protected long playlistId; + protected long trackId; + protected long userId; + + protected long timestamp; + private String type; + + public BaseTrackProgress(long playlistId, long trackId, long userId) { + this.playlistId = playlistId; + this.trackId = trackId; + this.userId = userId; + this.timestamp = System.currentTimeMillis(); + } + + public Long getTimestamp() { + return timestamp; + } + + public Long getUserId() { + return userId; + } + + public String getType() { + return type; + } + + public long getPlaylistId() { + return playlistId; + } + + public long getTrackId() { + return trackId; + } + + protected void setType(ProgressEntryType type) { + this.type = type.name(); + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java new file mode 100644 index 0000000..73b09a7 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java @@ -0,0 +1,37 @@ +package com.bivashy.backend.composer.dto.importing; + +public class PlaylistProgress extends BaseTrackProgress { + private String ytdlnStdout; + private int overallProgress; + private String status; + + public PlaylistProgress(long playlistId, long trackId, long userId) { + super(playlistId, trackId, userId); + this.setType(ProgressEntryType.PLAYLIST); + this.status = "LOADING"; + } + + public String getYtdlnStdout() { + return ytdlnStdout; + } + + public void setYtdlnStdout(String ytdlnStdout) { + this.ytdlnStdout = ytdlnStdout; + } + + public int getOverallProgress() { + return overallProgress; + } + + public void setOverallProgress(int overallProgress) { + this.overallProgress = overallProgress; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java new file mode 100644 index 0000000..65a2b06 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java @@ -0,0 +1,7 @@ +package com.bivashy.backend.composer.dto.importing; + +public enum ProgressEntryType { + PLAYLIST, + TRACK, + EXTERNAL_TRACK +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java new file mode 100644 index 0000000..cd2c957 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java @@ -0,0 +1,21 @@ +package com.bivashy.backend.composer.dto.importing; + +public class SingleTrackProgress extends BaseTrackProgress { + private String title; + private String format; + + public SingleTrackProgress(long playlistId, long trackId, long userId, String title, String format) { + super(playlistId, trackId, userId); + this.setType(ProgressEntryType.TRACK); + this.title = title; + this.format = format; + } + + public String getTitle() { + return title; + } + + public String getFormat() { + return format; + } +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java b/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java deleted file mode 100644 index da4b2ad..0000000 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.bivashy.backend.composer.dto.importing; - -public class TrackProgressDTO { - private long playlistId; - private long trackId; - private String trackTitle; - private String format; - private String sourceType; - private int progress; - private String metadata; - private Long timestamp; - private long userId; - - public TrackProgressDTO() { - } - - public TrackProgressDTO(long playlistId, long trackId, long userId) { - this.playlistId = playlistId; - this.trackId = trackId; - this.userId = userId; - this.timestamp = System.currentTimeMillis(); - } - - public TrackProgressDTO(long playlistId, - long trackId, - String trackTitle, - String format, - String sourceType, - int progress, - String metadata, - Long timestamp, - long userId) { - this.playlistId = playlistId; - this.trackId = trackId; - this.trackTitle = trackTitle; - this.format = format; - this.sourceType = sourceType; - this.progress = progress; - this.metadata = metadata; - this.timestamp = timestamp; - this.userId = userId; - } - - public long getPlaylistId() { - return playlistId; - } - - public void setPlaylistId(long playlistId) { - this.playlistId = playlistId; - } - - public long getTrackId() { - return trackId; - } - - public void setTrackId(long trackId) { - this.trackId = trackId; - } - - public String getTrackTitle() { - return trackTitle; - } - - public void setTrackTitle(String trackTitle) { - this.trackTitle = trackTitle; - } - - public String getFormat() { - return format; - } - - public void setFormat(String format) { - this.format = format; - } - - public String getSourceType() { - return sourceType; - } - - public void setSourceType(String sourceType) { - this.sourceType = sourceType; - } - - public int getProgress() { - return progress; - } - - public void setProgress(int progress) { - this.progress = progress; - } - - public String getMetadata() { - return metadata; - } - - public void setMetadata(String metadata) { - this.metadata = metadata; - } - - public Long getTimestamp() { - return timestamp; - } - - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - public long getUserId() { - return userId; - } - - public void setUserId(long userId) { - this.userId = userId; - } - -} diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java b/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java new file mode 100644 index 0000000..bf68275 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java @@ -0,0 +1,4 @@ +package com.bivashy.backend.composer.dto.track; + +public record AddYoutubeTrackRequest(String youtubeUrl) { +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java b/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java new file mode 100644 index 0000000..8f70d18 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java @@ -0,0 +1,9 @@ +package com.bivashy.backend.composer.dto.track.service; + +import com.bivashy.backend.composer.util.SimpleBlob; + +import io.soabase.recordbuilder.core.RecordBuilder; + +@RecordBuilder +public record AddLocalTrackParams(SimpleBlob blob, String ytdlpMetadata, boolean includeProgressHistory) { +} diff --git a/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java b/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java new file mode 100644 index 0000000..f5ba2a5 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java @@ -0,0 +1,11 @@ +package com.bivashy.backend.composer.exception; + +public class ImportTrackException extends Exception { + public ImportTrackException(String message, Exception cause) { + super(message, cause); + } + + public ImportTrackException(String message) { + super(message); + } +} diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java index 558506a..d9faaed 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java @@ -7,6 +7,8 @@ import java.util.Map; import org.springframework.http.MediaType; public interface AudioBlobStorageService { + String storeFolder(); + String store(InputStream inputStream); String store(byte[] data); @@ -15,6 +17,8 @@ public interface AudioBlobStorageService { String store(byte[] data, Map metadata); + String store(String key, byte[] data); + byte[] readRaw(String path) throws IOException; Blob read(String path); diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java index 1b96815..ee1a70b 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @Service @@ -36,6 +37,18 @@ public class AudioS3StorageService implements AudioBlobStorageService { this.tika = tika; } + @Override + public String storeFolder() { + String objectKey = newObjectName(); + if (!objectKey.endsWith("/")) + objectKey = objectKey + "/"; + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(), RequestBody.empty()); + return objectKey; + } + @Override public String store(InputStream inputStream) { return store(inputStream, Map.of()); @@ -61,7 +74,33 @@ public class AudioS3StorageService implements AudioBlobStorageService { .contentType(contentType) .metadata(metadata) .build(), RequestBody.fromBytes(data)); - return String.join("/", bucket, objectKey); + return objectKey; + } + + @Override + public String store(String key, byte[] data) { + try { + ResponseInputStream response = s3Client.getObject(GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .build()); + + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .contentType(response.response().contentType()) + .metadata(response.response().metadata()) + .build(), RequestBody.fromBytes(data)); + + return key; + } catch (NoSuchKeyException e) { + System.out.println("no existing found"); + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .build(), RequestBody.fromBytes(data)); + return key; + } } @Override @@ -71,9 +110,6 @@ public class AudioS3StorageService implements AudioBlobStorageService { @Override public Blob read(String path) { - if (path.startsWith(bucket + "/")) { - path = path.substring(bucket.length()); - } ResponseInputStream response = s3Client.getObject(GetObjectRequest.builder() .bucket(bucket) .key(path) diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackService.java b/src/main/java/com/bivashy/backend/composer/service/TrackService.java index c0f9471..40831ea 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackService.java @@ -1,56 +1,100 @@ package com.bivashy.backend.composer.service; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.server.ResponseStatusException; import com.bivashy.backend.composer.auth.CustomUserDetails; -import com.bivashy.backend.composer.dto.track.AddLocalTrackRequest; +import com.bivashy.backend.composer.dto.importing.SingleTrackProgress; +import com.bivashy.backend.composer.dto.track.AddYoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; import com.bivashy.backend.composer.model.SourceTypes; import com.bivashy.backend.composer.model.Track; import com.bivashy.backend.composer.model.TrackMetadata; import com.bivashy.backend.composer.model.TrackSource; import com.bivashy.backend.composer.repository.TrackRepository; import com.bivashy.backend.composer.service.MetadataParseService.Metadata; +import com.bivashy.backend.composer.service.importing.RedisProgressService; +import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jfposton.ytdlp.YtDlp; +import com.jfposton.ytdlp.YtDlpException; +import com.jfposton.ytdlp.YtDlpRequest; +import com.jfposton.ytdlp.mapper.VideoInfo; + +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; @Service public class TrackService { + private static final Logger logger = LoggerFactory.getLogger(TrackService.class); + public static final String DOWNLOADED_METADATA_FILE = "downloaded"; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final TrackRepository trackRepository; private final TrackSourceService trackSourceService; private final TrackMetadataService trackMetadataService; private final TrackPlaylistService trackPlaylistService; private final MetadataParseService metadataParseService; + private final RedisProgressService redisProgressService; + private final AudioS3StorageService s3StorageService; - public TrackService(TrackRepository trackRepository, - TrackSourceService trackSourceService, - TrackMetadataService trackMetadataService, - TrackPlaylistService trackPlaylistService, - MetadataParseService metadataParseService) { + public TrackService(TrackRepository trackRepository, TrackSourceService trackSourceService, + TrackMetadataService trackMetadataService, TrackPlaylistService trackPlaylistService, + MetadataParseService metadataParseService, RedisProgressService redisProgressService, + AudioS3StorageService s3StorageService) { this.trackRepository = trackRepository; this.trackSourceService = trackSourceService; this.trackMetadataService = trackMetadataService; this.trackPlaylistService = trackPlaylistService; this.metadataParseService = metadataParseService; + this.redisProgressService = redisProgressService; + this.s3StorageService = s3StorageService; } - public TrackResponse addLocalTrack(CustomUserDetails user, Long playlistId, AddLocalTrackRequest request) - throws IOException { - Optional metadata = metadataParseService.extractMetadata(request.source().getInputStream()); + public TrackResponse addLocalTrack(CustomUserDetails user, + long playlistId, + AddLocalTrackParams params) + throws ImportTrackException { + var request = params.blob(); + Optional metadata = Optional.empty(); + try (var inputStream = request.inputStream()) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + request.fileName()); + } String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); - TrackSource trackSource = trackSourceService.createTrackSource( - request.source().getBytes(), ffprobeJson, SourceTypes.FILE); + TrackSource trackSource; + try { + trackSource = trackSourceService.createLocalTrackSource( + request.body(), ffprobeJson, params.ytdlpMetadata(), SourceTypes.FILE); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } Track track = trackRepository.save(new Track(trackSource)); - String fileName = fileNameWithoutExtension(request.source().getOriginalFilename()); + String fileName = fileNameWithoutExtension(request.fileName()); String title = metadata.map(Metadata::title).orElse(fileName); String artist = metadata.map(Metadata::artist).orElse(null); int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); @@ -66,6 +110,12 @@ public class TrackService { if (metadata.isPresent()) { fileFormat = metadata.map(m -> m.formatName()).get(); } + + if (params.includeProgressHistory()) { + redisProgressService + .saveProgress(new SingleTrackProgress(playlistId, track.getId(), user.getId(), title, fileFormat)); + } + return new TrackResponse( track.getId(), title, @@ -76,6 +126,174 @@ public class TrackService { fileName); } + @Transactional + public List addYoutubeTrack(CustomUserDetails user, long playlistId, + AddYoutubeTrackRequest request) throws ImportTrackException { + List videoInfos = Collections.emptyList(); + try { + videoInfos = YtDlp.getVideoInfo(request.youtubeUrl()); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot `yt-dlp --dump-json` from " + request.youtubeUrl(), e); + } + + logger.info("videoinfos count {}", videoInfos.size()); + + if (videoInfos.size() == 0) { + throw new ImportTrackException("cannot find videoInfos"); + } + + if (videoInfos.size() == 1) { + try { + VideoInfo videoInfo = videoInfos.get(0); + + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + var ytDlpRequest = new YtDlpRequest(request.youtubeUrl(), temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + var response = YtDlp.execute(ytDlpRequest); + // TODO: write to RedisProgressService + + TrackResponse result = null; + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + + if (downloadedFiles.isEmpty()) + throw new ImportTrackException("yt-dlp didn't downloaded anything for " + request.youtubeUrl()); + + for (Path downloadedFile : downloadedFiles) { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new PathBlob(downloadedFile)) + .ytdlpMetadata(OBJECT_MAPPER.writeValueAsString(videoInfo)) + .includeProgressHistory(false) + .build(); + + result = addLocalTrack(user, + playlistId, + params); + Files.delete(downloadedFile); + } + } + Files.delete(temporaryFolder); + return List.of(result); + } catch (IOException e) { + throw new ImportTrackException("i/o during single youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + request.youtubeUrl(), e); + } + } + + TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceTypes.PLAYLIST); + return refreshYoutubePlaylist(playlistId, trackSource, videoInfos, request.youtubeUrl()); + } + + public List refreshYoutubePlaylist(long playlistId, TrackSource trackSource, + List videoInfos, + String youtubeUrl) throws ImportTrackException { + List result = new ArrayList<>(); + logger.info(trackSource.getSourceUrl()); + try { + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + logger.info("temporaryFolder created {}", temporaryFolder.toString()); + + String downloadedMetadataKey = trackSource.getSourceUrl() + DOWNLOADED_METADATA_FILE; + try { + var rawBody = s3StorageService + .readRaw(downloadedMetadataKey); + Files.write(temporaryFolder.resolve(DOWNLOADED_METADATA_FILE), rawBody); + } catch (NoSuchKeyException e) { + logger.warn(".downloaded metadata file was not found, ignoring"); + } + + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + ytDlpRequest.setOption("download-archive", DOWNLOADED_METADATA_FILE); + ytDlpRequest.setOption("extract-audio"); + ytDlpRequest.setOption("audio-quality", 0); + ytDlpRequest.setOption("audio-format", "best"); + ytDlpRequest.setOption("no-overwrites"); + var response = YtDlp.execute(ytDlpRequest); + logger.info("yt dlp response {}", response); + + // TODO: write to RedisProgressService + + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + logger.info("downloaded file count {}", downloadedFiles.size()); + + for (Path path : downloadedFiles) { + if (Files.isDirectory(path)) + continue; + boolean isMetadataFile = path.getFileName().toString().equals(DOWNLOADED_METADATA_FILE); + var body = Files.readAllBytes(path); + + if (isMetadataFile) { + s3StorageService.store(downloadedMetadataKey, body); + continue; + } + String fileName = fileNameWithoutExtension(path.getFileName().toString()); + VideoInfo videoInfo = videoInfos.stream() + .filter(v -> v.getId().equals(fileName)) + .findFirst() + .orElseThrow(); + + String audioKey = trackSource.getSourceUrl() + UUID.randomUUID().toString(); + + logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); + + audioKey = s3StorageService.store(audioKey, body); + Optional metadata = Optional.empty(); + + try (var inputStream = Files.newInputStream(path)) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + path.toString()); + } + String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); + + TrackSource playlistEntrySource; + try { + playlistEntrySource = trackSourceService.createLocalTrackSource( + body, ffprobeJson, OBJECT_MAPPER.writeValueAsString(videoInfo), SourceTypes.PLAYLIST); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } + + Track track = trackRepository.save(new Track(playlistEntrySource)); + + String title = videoInfo.getTitle(); + String artist = metadata.map(Metadata::artist).orElse(null); + int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); + // TODO: thumbnail + // TODO: Recognize music if the duration is less than five minutes + // (configurable), and if not, it is a playlist and should be marked as is + trackMetadataService.createTrackMetadata( + track, title, fileName, trackSource.getSourceUrl(), artist, null, durationSeconds); + + trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); + + String fileFormat = "unknown"; + if (metadata.isPresent()) { + fileFormat = metadata.map(m -> m.formatName()).get(); + } + + var trackResponse = new TrackResponse( + track.getId(), + title, + artist, + audioKey, + fileFormat, + durationSeconds, + fileName); + result.add(trackResponse); + } + } + return result; + } catch (IOException e) { + throw new ImportTrackException("i/o during playlist youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + youtubeUrl, e); + } + } + public List getPlaylistTracks(CustomUserDetails user, Long playlistId) { return trackPlaylistService.getPlaylistTracks(playlistId).stream() .map(pt -> { diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java index f9827e1..ad56bee 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java @@ -1,6 +1,7 @@ package com.bivashy.backend.composer.service; import java.time.LocalDateTime; +import java.util.HashMap; import java.util.Map; import org.springframework.stereotype.Service; @@ -12,6 +13,9 @@ import com.bivashy.backend.composer.repository.TrackSourceRepository; @Service public class TrackSourceService { + public static final String FFPROBE_METADATA_KEY = "ffprobe"; + public static final String YTDLP_METADATA_KEY = "ytdlp"; + private final TrackSourceRepository trackSourceRepository; private final SourceTypeRepository sourceTypeRepository; private final AudioBlobStorageService s3Service; @@ -24,10 +28,25 @@ public class TrackSourceService { this.s3Service = s3Service; } - public TrackSource createTrackSource(byte[] audioBytes, String ffprobeJson, String sourceType) { - String audioPath = s3Service.store(audioBytes, Map.of("ffprobe", ffprobeJson)); + public TrackSource createLocalTrackSource(byte[] audioBytes, + String ffprobeJson, + String ytdlpMetadata, + String sourceType) { + Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); + if (ytdlpMetadata != null) { + // TODO: Add tag or smth? + } + String audioPath = s3Service.store(audioBytes, metadata); + SourceType type = sourceTypeRepository.findByName(sourceType) .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); } + + public TrackSource createYoutubeTrackSource(String sourceType) { + String folderPath = s3Service.storeFolder(); + SourceType type = sourceTypeRepository.findByName(sourceType) + .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); + return trackSourceRepository.save(new TrackSource(folderPath, type, LocalDateTime.now())); + } } diff --git a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java index d3e70ab..1d9f4ad 100644 --- a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java +++ b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java @@ -1,7 +1,7 @@ package com.bivashy.backend.composer.service.importing; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; import com.bivashy.backend.composer.dto.importing.ImportTrackKey; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.data.redis.core.StringRedisTemplate; @@ -22,7 +22,7 @@ public class RedisProgressService { this.objectMapper = objectMapper; } - public void saveProgress(TrackProgressDTO progress) { + public void saveProgress(BaseTrackProgress progress) { try { String key = ImportTrackKey.progressKey(progress.getPlaylistId(), progress.getUserId()); String trackKey = ImportTrackKey.trackKey( @@ -44,16 +44,16 @@ public class RedisProgressService { } } - public List getPlaylistProgress(long playlistId, long userId) { + public List getPlaylistProgress(long playlistId, long userId) { try { String key = ImportTrackKey.progressKey(playlistId, userId); Map progressMap = redisTemplate.opsForHash().entries(key); - List progressList = new ArrayList<>(); + List progressList = new ArrayList<>(); for (Object value : progressMap.values()) { - TrackProgressDTO progress = objectMapper.readValue( + BaseTrackProgress progress = objectMapper.readValue( (String) value, - TrackProgressDTO.class); + BaseTrackProgress.class); progressList.add(progress); } @@ -65,13 +65,13 @@ public class RedisProgressService { } } - public TrackProgressDTO getTrackProgress(long playlistId, long trackId, long userId) { + public BaseTrackProgress getTrackProgress(long playlistId, long trackId, long userId) { try { String key = ImportTrackKey.trackKey(playlistId, trackId, userId); String progressJson = redisTemplate.opsForValue().get(key); if (progressJson != null) { - return objectMapper.readValue(progressJson, TrackProgressDTO.class); + return objectMapper.readValue(progressJson, BaseTrackProgress.class); } return null; } catch (Exception e) { @@ -79,7 +79,7 @@ public class RedisProgressService { } } - private void publishProgressUpdate(TrackProgressDTO progress) { + private void publishProgressUpdate(BaseTrackProgress progress) { try { String channel = ImportTrackKey.redisChannelKey(progress.getPlaylistId(), progress.getUserId()); String message = objectMapper.writeValueAsString(progress); diff --git a/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java b/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java new file mode 100644 index 0000000..4757cb0 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java @@ -0,0 +1,64 @@ +package com.bivashy.backend.composer.util; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.springframework.web.multipart.MultipartFile; + +public interface SimpleBlob { + InputStream inputStream() throws IOException; + + byte[] body() throws IOException; + + String fileName(); + + public static class MultipartBlob implements SimpleBlob { + private final MultipartFile multipartFile; + + public MultipartBlob(MultipartFile multipartFile) { + this.multipartFile = multipartFile; + } + + @Override + public InputStream inputStream() throws IOException { + return multipartFile.getInputStream(); + } + + @Override + public byte[] body() throws IOException { + return multipartFile.getBytes(); + } + + @Override + public String fileName() { + return multipartFile.getOriginalFilename(); + } + + } + + public static class PathBlob implements SimpleBlob { + private final Path path; + + public PathBlob(Path path) { + this.path = path; + } + + @Override + public InputStream inputStream() throws IOException { + return Files.newInputStream(path); + } + + @Override + public byte[] body() throws IOException { + return Files.readAllBytes(path); + } + + @Override + public String fileName() { + return path.getFileName().toString(); + } + + } +} -- 2.49.0 From b083e592f562e401839976c55136491a273df134 Mon Sep 17 00:00:00 2001 From: bivashy Date: Sun, 4 Jan 2026 05:30:10 +0500 Subject: [PATCH 2/5] Fix storing youtube files --- .../composer/controller/TrackController.java | 4 ++-- .../dto/track/AddYoutubeTrackRequest.java | 4 ---- .../composer/dto/track/YoutubeTrackRequest.java | 4 ++++ .../composer/service/AudioBlobStorageService.java | 2 +- .../composer/service/AudioS3StorageService.java | 5 +++-- .../backend/composer/service/TrackService.java | 15 +++++++-------- .../composer/service/TrackSourceService.java | 13 +++++++++++++ 7 files changed, 30 insertions(+), 17 deletions(-) delete mode 100644 src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java create mode 100644 src/main/java/com/bivashy/backend/composer/dto/track/YoutubeTrackRequest.java diff --git a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java index c249ebb..582416f 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java @@ -14,10 +14,10 @@ import org.springframework.web.bind.annotation.RestController; import com.bivashy.backend.composer.auth.CustomUserDetails; import com.bivashy.backend.composer.dto.track.AddLocalTrackRequest; -import com.bivashy.backend.composer.dto.track.AddYoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; +import com.bivashy.backend.composer.dto.track.YoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; import com.bivashy.backend.composer.exception.ImportTrackException; import com.bivashy.backend.composer.service.TrackService; @@ -47,7 +47,7 @@ public class TrackController { public ResponseEntity> addYoutubeTrack( @AuthenticationPrincipal CustomUserDetails user, @PathVariable long playlistId, - @RequestBody AddYoutubeTrackRequest request) throws ImportTrackException { + @RequestBody YoutubeTrackRequest request) throws ImportTrackException { List response = trackService.addYoutubeTrack(user, playlistId, request); return ResponseEntity.ok(response); } diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java b/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java deleted file mode 100644 index bf68275..0000000 --- a/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.bivashy.backend.composer.dto.track; - -public record AddYoutubeTrackRequest(String youtubeUrl) { -} diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/YoutubeTrackRequest.java b/src/main/java/com/bivashy/backend/composer/dto/track/YoutubeTrackRequest.java new file mode 100644 index 0000000..0ae8ad5 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/track/YoutubeTrackRequest.java @@ -0,0 +1,4 @@ +package com.bivashy.backend.composer.dto.track; + +public record YoutubeTrackRequest(String youtubeUrl) { +} diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java index d9faaed..3b65e82 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java @@ -17,7 +17,7 @@ public interface AudioBlobStorageService { String store(byte[] data, Map metadata); - String store(String key, byte[] data); + String store(String key, byte[] data, Map metadata); byte[] readRaw(String path) throws IOException; diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java index ee1a70b..5454518 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java @@ -78,7 +78,7 @@ public class AudioS3StorageService implements AudioBlobStorageService { } @Override - public String store(String key, byte[] data) { + public String store(String key, byte[] data, Map metadata) { try { ResponseInputStream response = s3Client.getObject(GetObjectRequest.builder() .bucket(bucket) @@ -89,7 +89,7 @@ public class AudioS3StorageService implements AudioBlobStorageService { .bucket(bucket) .key(key) .contentType(response.response().contentType()) - .metadata(response.response().metadata()) + .metadata(metadata) .build(), RequestBody.fromBytes(data)); return key; @@ -98,6 +98,7 @@ public class AudioS3StorageService implements AudioBlobStorageService { s3Client.putObject(PutObjectRequest.builder() .bucket(bucket) .key(key) + .metadata(metadata) .build(), RequestBody.fromBytes(data)); return key; } diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackService.java b/src/main/java/com/bivashy/backend/composer/service/TrackService.java index 40831ea..855474b 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackService.java @@ -1,12 +1,12 @@ package com.bivashy.backend.composer.service; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.stream.Stream; @@ -20,10 +20,10 @@ import org.springframework.web.server.ResponseStatusException; import com.bivashy.backend.composer.auth.CustomUserDetails; import com.bivashy.backend.composer.dto.importing.SingleTrackProgress; -import com.bivashy.backend.composer.dto.track.AddYoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; +import com.bivashy.backend.composer.dto.track.YoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; import com.bivashy.backend.composer.exception.ImportTrackException; @@ -128,7 +128,7 @@ public class TrackService { @Transactional public List addYoutubeTrack(CustomUserDetails user, long playlistId, - AddYoutubeTrackRequest request) throws ImportTrackException { + YoutubeTrackRequest request) throws ImportTrackException { List videoInfos = Collections.emptyList(); try { videoInfos = YtDlp.getVideoInfo(request.youtubeUrl()); @@ -226,7 +226,7 @@ public class TrackService { var body = Files.readAllBytes(path); if (isMetadataFile) { - s3StorageService.store(downloadedMetadataKey, body); + s3StorageService.store(downloadedMetadataKey, body, Map.of()); continue; } String fileName = fileNameWithoutExtension(path.getFileName().toString()); @@ -239,7 +239,6 @@ public class TrackService { logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); - audioKey = s3StorageService.store(audioKey, body); Optional metadata = Optional.empty(); try (var inputStream = Files.newInputStream(path)) { @@ -251,8 +250,8 @@ public class TrackService { TrackSource playlistEntrySource; try { - playlistEntrySource = trackSourceService.createLocalTrackSource( - body, ffprobeJson, OBJECT_MAPPER.writeValueAsString(videoInfo), SourceTypes.PLAYLIST); + playlistEntrySource = trackSourceService.createTrackSourceWithKey(audioKey, body, ffprobeJson, + OBJECT_MAPPER.writeValueAsString(videoInfo), SourceTypes.PLAYLIST); } catch (IOException e) { throw new ImportTrackException("cannot read blob body", e); } @@ -266,7 +265,7 @@ public class TrackService { // TODO: Recognize music if the duration is less than five minutes // (configurable), and if not, it is a playlist and should be marked as is trackMetadataService.createTrackMetadata( - track, title, fileName, trackSource.getSourceUrl(), artist, null, durationSeconds); + track, title, fileName, audioKey, artist, null, durationSeconds); trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java index ad56bee..eeebfff 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java @@ -43,6 +43,19 @@ public class TrackSourceService { return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); } + public TrackSource createTrackSourceWithKey(String key, byte[] audioBytes, String ffprobeJson, + String ytdlpMetadata, String sourceType) { + Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); + if (ytdlpMetadata != null) { + // TODO: Add tag or smth? + } + String audioPath = s3Service.store(key, audioBytes, metadata); + + SourceType type = sourceTypeRepository.findByName(sourceType) + .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); + return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); + } + public TrackSource createYoutubeTrackSource(String sourceType) { String folderPath = s3Service.storeFolder(); SourceType type = sourceTypeRepository.findByName(sourceType) -- 2.49.0 From dafcc6a64e784b4d77042d0c055760d3647dda0b Mon Sep 17 00:00:00 2001 From: bivashy Date: Mon, 5 Jan 2026 00:40:35 +0500 Subject: [PATCH 3/5] Implement `refresh` endpoint --- .gitignore | 2 + .../composer/controller/TrackController.java | 9 + .../backend/composer/model/Playlist.java | 2 +- .../composer/model/SourceMetadataType.java | 5 + .../backend/composer/model/SourceType.java | 46 +--- .../backend/composer/model/SourceTypes.java | 8 - .../backend/composer/model/TrackSource.java | 14 +- .../composer/model/TrackSourceMetadata.java | 47 ++++ .../bivashy/backend/composer/model/User.java | 2 +- .../repository/SourceTypeRepository.java | 11 - .../TrackSourceMetadataRepository.java | 18 ++ .../composer/service/TrackService.java | 173 ++------------ .../composer/service/TrackSourceService.java | 37 +-- .../composer/service/YoutubeTrackService.java | 216 ++++++++++++++++++ .../migration/V1_10__create_base_tables.sql | 117 +++++----- .../db/migration/V1_20__insert_enums.sql | 9 - 16 files changed, 405 insertions(+), 311 deletions(-) create mode 100644 src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java delete mode 100644 src/main/java/com/bivashy/backend/composer/model/SourceTypes.java create mode 100644 src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java delete mode 100644 src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java create mode 100644 src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java create mode 100644 src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java delete mode 100644 src/main/resources/db/migration/V1_20__insert_enums.sql diff --git a/.gitignore b/.gitignore index b9fcd77..22d23a8 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ build/ .vscode/ .env + +.sqruff diff --git a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java index 582416f..83ee40b 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java @@ -43,6 +43,15 @@ public class TrackController { return ResponseEntity.ok(response); } + @PostMapping(path = "/playlist/{playlistId}/track/youtube/refresh/{sourceId}") + public ResponseEntity> addYoutubeTrack( + @AuthenticationPrincipal CustomUserDetails user, + @PathVariable long playlistId, + @PathVariable long sourceId) throws ImportTrackException { + List response = trackService.refreshYoutubePlaylist(user, playlistId, sourceId); + return ResponseEntity.ok(response); + } + @PostMapping(path = "/playlist/{playlistId}/track/youtube") public ResponseEntity> addYoutubeTrack( @AuthenticationPrincipal CustomUserDetails user, diff --git a/src/main/java/com/bivashy/backend/composer/model/Playlist.java b/src/main/java/com/bivashy/backend/composer/model/Playlist.java index 061e0fc..ac95c60 100644 --- a/src/main/java/com/bivashy/backend/composer/model/Playlist.java +++ b/src/main/java/com/bivashy/backend/composer/model/Playlist.java @@ -28,7 +28,7 @@ public class Playlist { @JoinColumn(name = "owner_id", nullable = false) private User owner; - @Column(unique = true, nullable = false, length = 500) + @Column(unique = true, nullable = false) private String title; @Column(name = "created_at", nullable = false, updatable = false) diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java b/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java new file mode 100644 index 0000000..ae1c130 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java @@ -0,0 +1,5 @@ +package com.bivashy.backend.composer.model; + +public enum SourceMetadataType { + YOUTUBE +} diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceType.java b/src/main/java/com/bivashy/backend/composer/model/SourceType.java index d456aba..49d6d0b 100644 --- a/src/main/java/com/bivashy/backend/composer/model/SourceType.java +++ b/src/main/java/com/bivashy/backend/composer/model/SourceType.java @@ -1,47 +1,5 @@ package com.bivashy.backend.composer.model; -import java.util.HashSet; -import java.util.Set; - -import jakarta.persistence.CascadeType; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.OneToMany; -import jakarta.persistence.Table; - -@Entity -@Table(name = "source_type") -public class SourceType { - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column(nullable = false, length = 500) - private String name; - - @OneToMany(mappedBy = "sourceType", cascade = CascadeType.ALL, orphanRemoval = true) - private Set trackSources = new HashSet<>(); - - SourceType() { - } - - public SourceType(String name) { - this.name = name; - } - - public Long getId() { - return id; - } - - public String getName() { - return name; - } - - public Set getTrackSources() { - return trackSources; - } - +public enum SourceType { + VIDEO, PLAYLIST, PLAYLIST_ITEM, FILE, URL } diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java b/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java deleted file mode 100644 index f416568..0000000 --- a/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.bivashy.backend.composer.model; - -public class SourceTypes { - public static final String AUDIO = "VIDEO"; - public static final String PLAYLIST = "PLAYLIST"; - public static final String FILE = "FILE"; - public static final String URL = "URL"; -} diff --git a/src/main/java/com/bivashy/backend/composer/model/TrackSource.java b/src/main/java/com/bivashy/backend/composer/model/TrackSource.java index 2628303..7984f07 100644 --- a/src/main/java/com/bivashy/backend/composer/model/TrackSource.java +++ b/src/main/java/com/bivashy/backend/composer/model/TrackSource.java @@ -4,15 +4,16 @@ import java.time.LocalDateTime; import java.util.HashSet; import java.util.Set; +import org.hibernate.annotations.JdbcType; +import org.hibernate.dialect.PostgreSQLEnumJdbcType; + import jakarta.persistence.CascadeType; import jakarta.persistence.Column; import jakarta.persistence.Entity; -import jakarta.persistence.FetchType; +import jakarta.persistence.Enumerated; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; import jakarta.persistence.OneToMany; import jakarta.persistence.Table; @@ -23,11 +24,12 @@ public class TrackSource { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(name = "source_url", nullable = false, length = 500) + @Column(name = "source_url", nullable = false) private String sourceUrl; - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "source_type_id", nullable = false) + @Enumerated + @Column(name = "source_type", nullable = false) + @JdbcType(PostgreSQLEnumJdbcType.class) private SourceType sourceType; @Column(name = "last_fetched_at", nullable = false) diff --git a/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java b/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java new file mode 100644 index 0000000..4625f9e --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java @@ -0,0 +1,47 @@ +package com.bivashy.backend.composer.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; + +@Entity +@Table(name = "track_source_metadata") +public class TrackSourceMetadata { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "source_id", nullable = false, unique = true) + private TrackSource source; + + @Column(name = "url", nullable = false) + private String url; + + TrackSourceMetadata() { + } + + public TrackSourceMetadata(TrackSource source, String url) { + this.source = source; + this.url = url; + } + + public Long getId() { + return id; + } + + public TrackSource getSource() { + return source; + } + + public String getUrl() { + return url; + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/model/User.java b/src/main/java/com/bivashy/backend/composer/model/User.java index f0a049f..5241669 100644 --- a/src/main/java/com/bivashy/backend/composer/model/User.java +++ b/src/main/java/com/bivashy/backend/composer/model/User.java @@ -20,7 +20,7 @@ public class User { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(nullable = false, length = 500) + @Column(nullable = false) private String name; @Column(name = "created_at", nullable = false, updatable = false) diff --git a/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java b/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java deleted file mode 100644 index d73d4de..0000000 --- a/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.bivashy.backend.composer.repository; - -import java.util.Optional; - -import org.springframework.data.jpa.repository.JpaRepository; - -import com.bivashy.backend.composer.model.SourceType; - -public interface SourceTypeRepository extends JpaRepository { - Optional findByName(String name); -} diff --git a/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java b/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java new file mode 100644 index 0000000..95bd054 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java @@ -0,0 +1,18 @@ +package com.bivashy.backend.composer.repository; + +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.bivashy.backend.composer.model.TrackSourceMetadata; + +@Repository +public interface TrackSourceMetadataRepository extends JpaRepository { + @Query("SELECT tsm FROM TrackSourceMetadata tsm " + + "JOIN FETCH tsm.source " + + "WHERE tsm.source.id = :sourceId") + Optional findBySourceIdWithSource(@Param("sourceId") Long sourceId); +} diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackService.java b/src/main/java/com/bivashy/backend/composer/service/TrackService.java index 855474b..5aa48f2 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackService.java @@ -1,14 +1,13 @@ package com.bivashy.backend.composer.service; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.UUID; import java.util.stream.Stream; import org.slf4j.Logger; @@ -25,50 +24,42 @@ import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; import com.bivashy.backend.composer.dto.track.YoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; -import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; import com.bivashy.backend.composer.exception.ImportTrackException; -import com.bivashy.backend.composer.model.SourceTypes; +import com.bivashy.backend.composer.model.SourceType; import com.bivashy.backend.composer.model.Track; import com.bivashy.backend.composer.model.TrackMetadata; import com.bivashy.backend.composer.model.TrackSource; import com.bivashy.backend.composer.repository.TrackRepository; import com.bivashy.backend.composer.service.MetadataParseService.Metadata; import com.bivashy.backend.composer.service.importing.RedisProgressService; -import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; -import com.fasterxml.jackson.databind.ObjectMapper; import com.jfposton.ytdlp.YtDlp; import com.jfposton.ytdlp.YtDlpException; -import com.jfposton.ytdlp.YtDlpRequest; import com.jfposton.ytdlp.mapper.VideoInfo; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; - @Service public class TrackService { private static final Logger logger = LoggerFactory.getLogger(TrackService.class); public static final String DOWNLOADED_METADATA_FILE = "downloaded"; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final TrackRepository trackRepository; private final TrackSourceService trackSourceService; private final TrackMetadataService trackMetadataService; private final TrackPlaylistService trackPlaylistService; private final MetadataParseService metadataParseService; private final RedisProgressService redisProgressService; - private final AudioS3StorageService s3StorageService; + private final YoutubeTrackService youtubeTrackService; public TrackService(TrackRepository trackRepository, TrackSourceService trackSourceService, TrackMetadataService trackMetadataService, TrackPlaylistService trackPlaylistService, MetadataParseService metadataParseService, RedisProgressService redisProgressService, - AudioS3StorageService s3StorageService) { + YoutubeTrackService youtubeTrackService) { this.trackRepository = trackRepository; this.trackSourceService = trackSourceService; this.trackMetadataService = trackMetadataService; this.trackPlaylistService = trackPlaylistService; this.metadataParseService = metadataParseService; this.redisProgressService = redisProgressService; - this.s3StorageService = s3StorageService; + this.youtubeTrackService = youtubeTrackService; } public TrackResponse addLocalTrack(CustomUserDetails user, @@ -87,7 +78,7 @@ public class TrackService { TrackSource trackSource; try { trackSource = trackSourceService.createLocalTrackSource( - request.body(), ffprobeJson, params.ytdlpMetadata(), SourceTypes.FILE); + request.body(), ffprobeJson, params.ytdlpMetadata(), SourceType.FILE); } catch (IOException e) { throw new ImportTrackException("cannot read blob body", e); } @@ -126,6 +117,12 @@ public class TrackService { fileName); } + @Transactional + public List refreshYoutubePlaylist(CustomUserDetails user, long playlistId, long sourceId) + throws ImportTrackException { + return youtubeTrackService.refreshYoutubePlaylist(playlistId, sourceId); + } + @Transactional public List addYoutubeTrack(CustomUserDetails user, long playlistId, YoutubeTrackRequest request) throws ImportTrackException { @@ -145,34 +142,17 @@ public class TrackService { if (videoInfos.size() == 1) { try { VideoInfo videoInfo = videoInfos.get(0); - Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); - var ytDlpRequest = new YtDlpRequest(request.youtubeUrl(), temporaryFolder.toAbsolutePath().toString()); - ytDlpRequest.setOption("output", "%(id)s"); - var response = YtDlp.execute(ytDlpRequest); - // TODO: write to RedisProgressService - TrackResponse result = null; + var params = youtubeTrackService.downloadYoutubeTrack(temporaryFolder, videoInfo, + request.youtubeUrl()); + TrackResponse result = addLocalTrack(user, playlistId, params); + try (Stream pathStream = Files.walk(temporaryFolder)) { - List downloadedFiles = Files.walk(temporaryFolder).toList(); - - if (downloadedFiles.isEmpty()) - throw new ImportTrackException("yt-dlp didn't downloaded anything for " + request.youtubeUrl()); - - for (Path downloadedFile : downloadedFiles) { - var params = AddLocalTrackParamsBuilder.builder() - .blob(new PathBlob(downloadedFile)) - .ytdlpMetadata(OBJECT_MAPPER.writeValueAsString(videoInfo)) - .includeProgressHistory(false) - .build(); - - result = addLocalTrack(user, - playlistId, - params); - Files.delete(downloadedFile); - } + pathStream.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); } - Files.delete(temporaryFolder); return List.of(result); } catch (IOException e) { throw new ImportTrackException("i/o during single youtube video downloading", e); @@ -181,116 +161,9 @@ public class TrackService { } } - TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceTypes.PLAYLIST); - return refreshYoutubePlaylist(playlistId, trackSource, videoInfos, request.youtubeUrl()); - } - - public List refreshYoutubePlaylist(long playlistId, TrackSource trackSource, - List videoInfos, - String youtubeUrl) throws ImportTrackException { - List result = new ArrayList<>(); - logger.info(trackSource.getSourceUrl()); - try { - Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); - logger.info("temporaryFolder created {}", temporaryFolder.toString()); - - String downloadedMetadataKey = trackSource.getSourceUrl() + DOWNLOADED_METADATA_FILE; - try { - var rawBody = s3StorageService - .readRaw(downloadedMetadataKey); - Files.write(temporaryFolder.resolve(DOWNLOADED_METADATA_FILE), rawBody); - } catch (NoSuchKeyException e) { - logger.warn(".downloaded metadata file was not found, ignoring"); - } - - var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); - ytDlpRequest.setOption("output", "%(id)s"); - ytDlpRequest.setOption("download-archive", DOWNLOADED_METADATA_FILE); - ytDlpRequest.setOption("extract-audio"); - ytDlpRequest.setOption("audio-quality", 0); - ytDlpRequest.setOption("audio-format", "best"); - ytDlpRequest.setOption("no-overwrites"); - var response = YtDlp.execute(ytDlpRequest); - logger.info("yt dlp response {}", response); - - // TODO: write to RedisProgressService - - try (Stream pathStream = Files.walk(temporaryFolder)) { - List downloadedFiles = Files.walk(temporaryFolder).toList(); - logger.info("downloaded file count {}", downloadedFiles.size()); - - for (Path path : downloadedFiles) { - if (Files.isDirectory(path)) - continue; - boolean isMetadataFile = path.getFileName().toString().equals(DOWNLOADED_METADATA_FILE); - var body = Files.readAllBytes(path); - - if (isMetadataFile) { - s3StorageService.store(downloadedMetadataKey, body, Map.of()); - continue; - } - String fileName = fileNameWithoutExtension(path.getFileName().toString()); - VideoInfo videoInfo = videoInfos.stream() - .filter(v -> v.getId().equals(fileName)) - .findFirst() - .orElseThrow(); - - String audioKey = trackSource.getSourceUrl() + UUID.randomUUID().toString(); - - logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); - - Optional metadata = Optional.empty(); - - try (var inputStream = Files.newInputStream(path)) { - metadata = metadataParseService.extractMetadata(inputStream); - } catch (IOException e) { - throw new ImportTrackException("cannot extract metadata from " + path.toString()); - } - String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); - - TrackSource playlistEntrySource; - try { - playlistEntrySource = trackSourceService.createTrackSourceWithKey(audioKey, body, ffprobeJson, - OBJECT_MAPPER.writeValueAsString(videoInfo), SourceTypes.PLAYLIST); - } catch (IOException e) { - throw new ImportTrackException("cannot read blob body", e); - } - - Track track = trackRepository.save(new Track(playlistEntrySource)); - - String title = videoInfo.getTitle(); - String artist = metadata.map(Metadata::artist).orElse(null); - int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); - // TODO: thumbnail - // TODO: Recognize music if the duration is less than five minutes - // (configurable), and if not, it is a playlist and should be marked as is - trackMetadataService.createTrackMetadata( - track, title, fileName, audioKey, artist, null, durationSeconds); - - trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); - - String fileFormat = "unknown"; - if (metadata.isPresent()) { - fileFormat = metadata.map(m -> m.formatName()).get(); - } - - var trackResponse = new TrackResponse( - track.getId(), - title, - artist, - audioKey, - fileFormat, - durationSeconds, - fileName); - result.add(trackResponse); - } - } - return result; - } catch (IOException e) { - throw new ImportTrackException("i/o during playlist youtube video downloading", e); - } catch (YtDlpException e) { - throw new ImportTrackException("cannot download youtube video " + youtubeUrl, e); - } + TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceType.PLAYLIST, + request.youtubeUrl()); + return youtubeTrackService.refreshYoutubePlaylist(playlistId, trackSource, videoInfos, request.youtubeUrl()); } public List getPlaylistTracks(CustomUserDetails user, Long playlistId) { diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java index eeebfff..2e45621 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java @@ -3,12 +3,14 @@ package com.bivashy.backend.composer.service; import java.time.LocalDateTime; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import org.springframework.stereotype.Service; import com.bivashy.backend.composer.model.SourceType; import com.bivashy.backend.composer.model.TrackSource; -import com.bivashy.backend.composer.repository.SourceTypeRepository; +import com.bivashy.backend.composer.model.TrackSourceMetadata; +import com.bivashy.backend.composer.repository.TrackSourceMetadataRepository; import com.bivashy.backend.composer.repository.TrackSourceRepository; @Service @@ -17,49 +19,50 @@ public class TrackSourceService { public static final String YTDLP_METADATA_KEY = "ytdlp"; private final TrackSourceRepository trackSourceRepository; - private final SourceTypeRepository sourceTypeRepository; + private final TrackSourceMetadataRepository trackSourceMetadataRepository; private final AudioBlobStorageService s3Service; public TrackSourceService(TrackSourceRepository trackSourceRepository, - SourceTypeRepository sourceTypeRepository, - AudioBlobStorageService s3Service) { + TrackSourceMetadataRepository trackSourceMetadataRepository, AudioBlobStorageService s3Service) { this.trackSourceRepository = trackSourceRepository; - this.sourceTypeRepository = sourceTypeRepository; + this.trackSourceMetadataRepository = trackSourceMetadataRepository; this.s3Service = s3Service; } public TrackSource createLocalTrackSource(byte[] audioBytes, String ffprobeJson, String ytdlpMetadata, - String sourceType) { + SourceType sourceType) { Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); if (ytdlpMetadata != null) { // TODO: Add tag or smth? } String audioPath = s3Service.store(audioBytes, metadata); - SourceType type = sourceTypeRepository.findByName(sourceType) - .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); - return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); + return trackSourceRepository.save(new TrackSource(audioPath, sourceType, LocalDateTime.now())); } public TrackSource createTrackSourceWithKey(String key, byte[] audioBytes, String ffprobeJson, - String ytdlpMetadata, String sourceType) { + String ytdlpMetadata, SourceType sourceType) { Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); if (ytdlpMetadata != null) { // TODO: Add tag or smth? } String audioPath = s3Service.store(key, audioBytes, metadata); - SourceType type = sourceTypeRepository.findByName(sourceType) - .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); - return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); + return trackSourceRepository.save(new TrackSource(audioPath, sourceType, LocalDateTime.now())); } - public TrackSource createYoutubeTrackSource(String sourceType) { + public TrackSource createYoutubeTrackSource(SourceType sourceType, String youtubeUrl) { String folderPath = s3Service.storeFolder(); - SourceType type = sourceTypeRepository.findByName(sourceType) - .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); - return trackSourceRepository.save(new TrackSource(folderPath, type, LocalDateTime.now())); + TrackSource trackSource = trackSourceRepository + .save(new TrackSource(folderPath, sourceType, LocalDateTime.now())); + trackSourceMetadataRepository.save(new TrackSourceMetadata(trackSource, youtubeUrl)); + return trackSource; } + + public Optional findWithMetadata(long sourceId) { + return trackSourceMetadataRepository.findBySourceIdWithSource(sourceId); + } + } diff --git a/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java new file mode 100644 index 0000000..ded3db1 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java @@ -0,0 +1,216 @@ +package com.bivashy.backend.composer.service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.bivashy.backend.composer.dto.track.TrackResponse; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; +import com.bivashy.backend.composer.model.SourceType; +import com.bivashy.backend.composer.model.Track; +import com.bivashy.backend.composer.model.TrackSource; +import com.bivashy.backend.composer.model.TrackSourceMetadata; +import com.bivashy.backend.composer.repository.TrackRepository; +import com.bivashy.backend.composer.service.MetadataParseService.Metadata; +import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jfposton.ytdlp.YtDlp; +import com.jfposton.ytdlp.YtDlpException; +import com.jfposton.ytdlp.YtDlpRequest; +import com.jfposton.ytdlp.mapper.VideoInfo; + +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; + +@Service +public class YoutubeTrackService { + private static final Logger logger = LoggerFactory.getLogger(YoutubeTrackService.class); + public static final String DOWNLOADED_METADATA_FILE = "downloaded"; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final AudioS3StorageService s3StorageService; + private final MetadataParseService metadataParseService; + private final TrackRepository trackRepository; + private final TrackMetadataService trackMetadataService; + private final TrackPlaylistService trackPlaylistService; + private final TrackSourceService trackSourceService; + + public YoutubeTrackService(AudioS3StorageService s3StorageService, MetadataParseService metadataParseService, + TrackRepository trackRepository, TrackMetadataService trackMetadataService, + TrackPlaylistService trackPlaylistService, TrackSourceService trackSourceService) { + this.s3StorageService = s3StorageService; + this.metadataParseService = metadataParseService; + this.trackRepository = trackRepository; + this.trackMetadataService = trackMetadataService; + this.trackPlaylistService = trackPlaylistService; + this.trackSourceService = trackSourceService; + } + + public AddLocalTrackParams downloadYoutubeTrack(Path temporaryFolder, VideoInfo videoInfo, String youtubeUrl) + throws IOException, YtDlpException, ImportTrackException { + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + var response = YtDlp.execute(ytDlpRequest); + // TODO: write to RedisProgressService + + TrackResponse result = null; + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + + if (downloadedFiles.isEmpty()) + throw new ImportTrackException("yt-dlp didn't downloaded anything for " + youtubeUrl); + + for (Path downloadedFile : downloadedFiles) { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new PathBlob(downloadedFile)) + .ytdlpMetadata(OBJECT_MAPPER.writeValueAsString(videoInfo)) + .includeProgressHistory(false) + .build(); + + return params; + } + } + throw new ImportTrackException("cannot download any youtube track"); + } + + public List refreshYoutubePlaylist(long playlistId, long sourceId) throws ImportTrackException { + Optional trackSourceMetadataOpt = trackSourceService.findWithMetadata(sourceId); + if (trackSourceMetadataOpt.isEmpty()) + throw new ImportTrackException("cannot find track source with metadata with id " + sourceId); + TrackSourceMetadata trackSourceMetadata = trackSourceMetadataOpt.get(); + String youtubeUrl = trackSourceMetadata.getUrl(); + + List videoInfos = Collections.emptyList(); + try { + videoInfos = YtDlp.getVideoInfo(youtubeUrl); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot `yt-dlp --dump-json` from " + youtubeUrl, e); + } + return refreshYoutubePlaylist(playlistId, trackSourceMetadata.getSource(), videoInfos, youtubeUrl); + } + + public List refreshYoutubePlaylist(long playlistId, TrackSource trackSource, + List videoInfos, + String youtubeUrl) throws ImportTrackException { + List result = new ArrayList<>(); + logger.info(trackSource.getSourceUrl()); + try { + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + logger.info("temporaryFolder created {}", temporaryFolder.toString()); + + String downloadedMetadataKey = trackSource.getSourceUrl() + DOWNLOADED_METADATA_FILE; + try { + var rawBody = s3StorageService + .readRaw(downloadedMetadataKey); + Files.write(temporaryFolder.resolve(DOWNLOADED_METADATA_FILE), rawBody); + } catch (NoSuchKeyException e) { + logger.warn(".downloaded metadata file was not found, ignoring"); + } + + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + ytDlpRequest.setOption("download-archive", DOWNLOADED_METADATA_FILE); + ytDlpRequest.setOption("extract-audio"); + ytDlpRequest.setOption("audio-quality", 0); + ytDlpRequest.setOption("audio-format", "best"); + ytDlpRequest.setOption("no-overwrites"); + var response = YtDlp.execute(ytDlpRequest); + logger.info("yt dlp response {}", response); + + // TODO: write to RedisProgressService + + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + logger.info("downloaded file count {}", downloadedFiles.size()); + + for (Path path : downloadedFiles) { + if (Files.isDirectory(path)) + continue; + boolean isMetadataFile = path.getFileName().toString().equals(DOWNLOADED_METADATA_FILE); + var body = Files.readAllBytes(path); + + if (isMetadataFile) { + s3StorageService.store(downloadedMetadataKey, body, Map.of()); + continue; + } + String fileName = fileNameWithoutExtension(path.getFileName().toString()); + VideoInfo videoInfo = videoInfos.stream() + .filter(v -> v.getId().equals(fileName)) + .findFirst() + .orElseThrow(); + + String audioKey = trackSource.getSourceUrl() + UUID.randomUUID().toString(); + + logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); + + Optional metadata = Optional.empty(); + + try (var inputStream = Files.newInputStream(path)) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + path.toString()); + } + String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); + + TrackSource playlistEntrySource; + try { + playlistEntrySource = trackSourceService.createTrackSourceWithKey(audioKey, body, ffprobeJson, + OBJECT_MAPPER.writeValueAsString(videoInfo), SourceType.PLAYLIST_ITEM); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } + + Track track = trackRepository.save(new Track(playlistEntrySource)); + + String title = videoInfo.getTitle(); + String artist = metadata.map(Metadata::artist).orElse(null); + int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); + // TODO: thumbnail + // TODO: Recognize music if the duration is less than five minutes + // (configurable), and if not, it is a playlist and should be marked as is + trackMetadataService.createTrackMetadata( + track, title, fileName, audioKey, artist, null, durationSeconds); + + trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); + + String fileFormat = "unknown"; + if (metadata.isPresent()) { + fileFormat = metadata.map(m -> m.formatName()).get(); + } + + var trackResponse = new TrackResponse( + track.getId(), + title, + artist, + audioKey, + fileFormat, + durationSeconds, + fileName); + result.add(trackResponse); + } + } + return result; + } catch (IOException e) { + throw new ImportTrackException("i/o during playlist youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + youtubeUrl, e); + } + } + + private String fileNameWithoutExtension(String fileName) { + return fileName.replaceFirst("[.][^.]+$", ""); + } + +} diff --git a/src/main/resources/db/migration/V1_10__create_base_tables.sql b/src/main/resources/db/migration/V1_10__create_base_tables.sql index 5c4c53b..262d2b7 100644 --- a/src/main/resources/db/migration/V1_10__create_base_tables.sql +++ b/src/main/resources/db/migration/V1_10__create_base_tables.sql @@ -1,80 +1,69 @@ -CREATE TABLE IF NOT EXISTS "users" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "name" varchar(500) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW() +CREATE TABLE IF NOT EXISTS users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE TABLE IF NOT EXISTS "source_type" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "name" varchar(500) NOT NULL +CREATE TYPE source_type_enum AS ENUM ( + 'VIDEO', 'PLAYLIST', 'PLAYLIST_ITEM', 'FILE', 'URL' ); -CREATE TABLE IF NOT EXISTS "track_source" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "source_url" varchar(500) NOT NULL, - "source_type_id" bigint NOT NULL, - "last_fetched_at" timestamp NOT NULL DEFAULT NOW(), - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_track_source_source_type_id" - FOREIGN KEY ("source_type_id") REFERENCES "source_type" ("id") +CREATE TABLE IF NOT EXISTS track_source ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_url TEXT NOT NULL, + source_type SOURCE_TYPE_ENUM NOT NULL, + last_fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE TABLE IF NOT EXISTS "track" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "source_id" bigint NOT NULL, - CONSTRAINT "fk_track_source_id" - FOREIGN KEY ("source_id") REFERENCES "track_source" ("id") +CREATE TABLE IF NOT EXISTS track_source_metadata ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_id BIGINT NOT NULL UNIQUE REFERENCES track_source ( + id + ) ON DELETE CASCADE, + url TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS "track_metadata" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "track_id" bigint NOT NULL, - "title" varchar(500) NOT NULL, - "file_name" varchar(500) NOT NULL, - "audio_path" varchar(500) NOT NULL, - "artist" varchar(500), - "thumbnail_path" varchar(500), - "duration_seconds" integer, - CONSTRAINT "fk_track_metadata_track_id" - FOREIGN KEY ("track_id") REFERENCES "track" ("id") +CREATE TABLE IF NOT EXISTS track ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES track_source (id) ON DELETE RESTRICT ); -CREATE TABLE IF NOT EXISTS "playlist" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "owner_id" bigint NOT NULL, - "title" varchar(500) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_playlist_owner_id" - FOREIGN KEY ("owner_id") REFERENCES "users" ("id"), - CONSTRAINT "uq_playlist_owner_title" - UNIQUE ("owner_id", "title") +CREATE TABLE IF NOT EXISTS track_metadata ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + title TEXT NOT NULL, + file_name TEXT NOT NULL, + audio_path TEXT NOT NULL, + artist TEXT, + thumbnail_path TEXT, + duration_seconds INTEGER ); -CREATE TABLE IF NOT EXISTS "playlist_track" ( - "playlist_id" bigint NOT NULL, - "track_id" bigint NOT NULL, - "order_index" numeric NOT NULL, - CONSTRAINT "pk_playlist_track_new" PRIMARY KEY ("playlist_id", "track_id"), - CONSTRAINT "fk_playlist_track_playlist_id_new" - FOREIGN KEY ("playlist_id") REFERENCES "playlist" ("id"), - CONSTRAINT "fk_playlist_track_track_id_new" - FOREIGN KEY ("track_id") REFERENCES "track" ("id") +CREATE TABLE IF NOT EXISTS playlist ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + owner_id BIGINT NOT NULL REFERENCES users (id) ON DELETE CASCADE, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (owner_id, title) ); -CREATE TABLE IF NOT EXISTS "track_version" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "track_id" bigint NOT NULL, - "metadata_id" bigint NOT NULL, - "source_id" bigint NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_track_version_track_id" - FOREIGN KEY ("track_id") REFERENCES "track" ("id"), - CONSTRAINT "fk_track_version_metadata_id" - FOREIGN KEY ("metadata_id") REFERENCES "track_metadata" ("id"), - CONSTRAINT "fk_track_version_source_id" - FOREIGN KEY ("source_id") REFERENCES "track_source" ("id") +CREATE TABLE IF NOT EXISTS playlist_track ( + playlist_id BIGINT NOT NULL REFERENCES playlist (id) ON DELETE CASCADE, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + order_index DECIMAL NOT NULL, + PRIMARY KEY (playlist_id, track_id) ); +CREATE TABLE IF NOT EXISTS track_version ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + metadata_id BIGINT NOT NULL REFERENCES track_metadata ( + id + ) ON DELETE CASCADE, + source_id BIGINT NOT NULL REFERENCES track_source (id) ON DELETE RESTRICT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/src/main/resources/db/migration/V1_20__insert_enums.sql b/src/main/resources/db/migration/V1_20__insert_enums.sql deleted file mode 100644 index 652794c..0000000 --- a/src/main/resources/db/migration/V1_20__insert_enums.sql +++ /dev/null @@ -1,9 +0,0 @@ -INSERT INTO "source_type" ("id", "name") -OVERRIDING SYSTEM VALUE -VALUES - (1, 'VIDEO'), - (2, 'PLAYLIST'), - (3, 'FILE'), - (4, 'URL') -ON CONFLICT ("id") DO NOTHING; - -- 2.49.0 From d45b2a96412c2b5038fdbc2c7905597540f88d3e Mon Sep 17 00:00:00 2001 From: bivashy Date: Thu, 8 Jan 2026 01:09:41 +0500 Subject: [PATCH 4/5] Improve `ProgressSSEController` OpenAPI, and progress --- pom.xml | 7 ++- .../importing/ProgressSSEController.java | 22 +++---- .../dto/importing/BaseTrackProgress.java | 10 ++-- .../dto/importing/ImportTrackKey.java | 4 +- .../dto/importing/PlaylistProgress.java | 23 ++++++-- .../dto/importing/ProgressStatus.java | 5 ++ .../dto/importing/SingleTrackProgress.java | 8 ++- .../composer/service/TrackService.java | 8 ++- .../composer/service/YoutubeTrackService.java | 42 +++++++++++--- .../importing/RedisMessageSubscriber.java | 27 ++++++--- .../importing/RedisProgressService.java | 58 ++++++++++++++++--- 11 files changed, 159 insertions(+), 55 deletions(-) create mode 100644 src/main/java/com/bivashy/backend/composer/dto/importing/ProgressStatus.java diff --git a/pom.xml b/pom.xml index ae9b992..7e96b67 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 3.2.3 2.8.5 2024.08.29 - 2.0.6 + 2.0.7 51 @@ -121,6 +121,11 @@ springdoc-openapi-starter-webmvc-ui ${springdoc-openapi.version} + + org.springdoc + springdoc-openapi-starter-webflux-ui + ${springdoc-openapi.version} + com.github.kokorin.jaffree jaffree diff --git a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java index dfdec00..b6d4b92 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java @@ -4,8 +4,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; -import org.springframework.http.codec.ServerSentEvent; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -16,7 +17,6 @@ import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; import com.bivashy.backend.composer.dto.importing.ImportTrackKey; import com.bivashy.backend.composer.service.importing.RedisMessageSubscriber; import com.bivashy.backend.composer.service.importing.RedisProgressService; -import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.http.HttpServletResponse; import reactor.core.publisher.Flux; @@ -24,10 +24,11 @@ import reactor.core.publisher.Sinks; @RestController public class ProgressSSEController { + private static final Logger logger = LoggerFactory.getLogger(ProgressSSEController.class); private final RedisProgressService redisProgressService; private final RedisMessageSubscriber redisSubscriber; - private final Map> sinks = new ConcurrentHashMap<>(); + private final Map> sinks = new ConcurrentHashMap<>(); public ProgressSSEController(RedisProgressService redisProgressService, RedisMessageSubscriber redisSubscriber) { @@ -36,7 +37,7 @@ public class ProgressSSEController { } @GetMapping(value = "/importing/stream/{playlistId}", produces = MediaType.TEXT_EVENT_STREAM_VALUE) - public Flux> streamProgress( + public Flux streamProgress( @PathVariable long playlistId, @AuthenticationPrincipal CustomUserDetails user, HttpServletResponse response) { @@ -48,8 +49,8 @@ public class ProgressSSEController { String connectionKey = ImportTrackKey.subscriptionKey(playlistId, userId); - Sinks.Many sink = sinks.computeIfAbsent(connectionKey, k -> { - Sinks.Many newSink = Sinks.many().replay().latest(); + Sinks.Many sink = sinks.computeIfAbsent(connectionKey, k -> { + Sinks.Many newSink = Sinks.many().replay().latest(); redisSubscriber.subscribeToPlaylist(playlistId, userId, message -> { newSink.tryEmitNext(message); @@ -61,19 +62,14 @@ public class ProgressSSEController { redisProgressService.addActiveConnection(playlistId, userId); return sink.asFlux() - .map(data -> ServerSentEvent.builder() - .data(data) - .event("progress-update") - .build()) .doFirst(() -> { try { List existingProgresses = redisProgressService.getPlaylistProgress( playlistId, userId); - ObjectMapper mapper = new ObjectMapper(); for (BaseTrackProgress progress : existingProgresses) { - sink.tryEmitNext(mapper.writeValueAsString(progress)); + sink.tryEmitNext(progress); } } catch (Exception e) { e.printStackTrace(); @@ -92,7 +88,7 @@ public class ProgressSSEController { } private void cleanupConnection(Long playlistId, long userId, - Sinks.Many sink, String connectionKey) { + Sinks.Many sink, String connectionKey) { try { redisProgressService.removeActiveConnection(playlistId, userId); redisSubscriber.unsubscribeFromPlaylist(playlistId, userId); diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java index da33855..e33b59e 100644 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java @@ -10,15 +10,15 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; }) public abstract class BaseTrackProgress { protected long playlistId; - protected long trackId; + protected long trackSourceId; protected long userId; protected long timestamp; private String type; - public BaseTrackProgress(long playlistId, long trackId, long userId) { + public BaseTrackProgress(long playlistId, long trackSourceId, long userId) { this.playlistId = playlistId; - this.trackId = trackId; + this.trackSourceId = trackSourceId; this.userId = userId; this.timestamp = System.currentTimeMillis(); } @@ -39,8 +39,8 @@ public abstract class BaseTrackProgress { return playlistId; } - public long getTrackId() { - return trackId; + public long getTrackSourceId() { + return trackSourceId; } protected void setType(ProgressEntryType type) { diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java b/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java index 6e4e4f2..6ac8ac8 100644 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java @@ -5,8 +5,8 @@ public class ImportTrackKey { return String.format("progress:%d:%d", userId, playlistId); } - public static String trackKey(long playlistId, long trackId, long userId) { - return String.format("track:%d:%d:%d", userId, playlistId, trackId); + public static String trackKey(long playlistId, long trackSourceId, long userId) { + return String.format("track:%d:%d:%d", userId, playlistId, trackSourceId); } public static String redisChannelKey(long playlistId, long userId) { diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java index 73b09a7..7d90293 100644 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java @@ -3,12 +3,18 @@ package com.bivashy.backend.composer.dto.importing; public class PlaylistProgress extends BaseTrackProgress { private String ytdlnStdout; private int overallProgress; - private String status; + private int trackCount; + private ProgressStatus status; - public PlaylistProgress(long playlistId, long trackId, long userId) { - super(playlistId, trackId, userId); + PlaylistProgress() { + super(0, 0, 0); + } + + public PlaylistProgress(long playlistId, long trackSourceId, long userId, int trackCount) { + super(playlistId, trackSourceId, userId); this.setType(ProgressEntryType.PLAYLIST); - this.status = "LOADING"; + this.status = ProgressStatus.LOADING; + this.trackCount = trackCount; } public String getYtdlnStdout() { @@ -27,11 +33,16 @@ public class PlaylistProgress extends BaseTrackProgress { this.overallProgress = overallProgress; } - public String getStatus() { + public ProgressStatus getStatus() { return status; } - public void setStatus(String status) { + public void setStatus(ProgressStatus status) { this.status = status; } + + public int getTrackCount() { + return trackCount; + } + } diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressStatus.java b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressStatus.java new file mode 100644 index 0000000..68a79c1 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressStatus.java @@ -0,0 +1,5 @@ +package com.bivashy.backend.composer.dto.importing; + +public enum ProgressStatus { + LOADING, FINISHED +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java index cd2c957..a98f018 100644 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java @@ -4,8 +4,12 @@ public class SingleTrackProgress extends BaseTrackProgress { private String title; private String format; - public SingleTrackProgress(long playlistId, long trackId, long userId, String title, String format) { - super(playlistId, trackId, userId); + SingleTrackProgress() { + super(0, 0, 0); + } + + public SingleTrackProgress(long playlistId, long trackSourceId, long userId, String title, String format) { + super(playlistId, trackSourceId, userId); this.setType(ProgressEntryType.TRACK); this.title = title; this.format = format; diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackService.java b/src/main/java/com/bivashy/backend/composer/service/TrackService.java index 5aa48f2..3ab2fdd 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackService.java @@ -104,7 +104,8 @@ public class TrackService { if (params.includeProgressHistory()) { redisProgressService - .saveProgress(new SingleTrackProgress(playlistId, track.getId(), user.getId(), title, fileFormat)); + .saveProgress( + new SingleTrackProgress(playlistId, trackSource.getId(), user.getId(), title, fileFormat)); } return new TrackResponse( @@ -120,7 +121,7 @@ public class TrackService { @Transactional public List refreshYoutubePlaylist(CustomUserDetails user, long playlistId, long sourceId) throws ImportTrackException { - return youtubeTrackService.refreshYoutubePlaylist(playlistId, sourceId); + return youtubeTrackService.refreshYoutubePlaylist(user, playlistId, sourceId); } @Transactional @@ -163,7 +164,8 @@ public class TrackService { TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceType.PLAYLIST, request.youtubeUrl()); - return youtubeTrackService.refreshYoutubePlaylist(playlistId, trackSource, videoInfos, request.youtubeUrl()); + return youtubeTrackService.refreshYoutubePlaylist(user.getId(), playlistId, trackSource, videoInfos, + request.youtubeUrl()); } public List getPlaylistTracks(CustomUserDetails user, Long playlistId) { diff --git a/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java index ded3db1..a6ae2e4 100644 --- a/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java @@ -15,6 +15,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import com.bivashy.backend.composer.auth.CustomUserDetails; +import com.bivashy.backend.composer.dto.importing.PlaylistProgress; +import com.bivashy.backend.composer.dto.importing.ProgressStatus; import com.bivashy.backend.composer.dto.track.TrackResponse; import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; @@ -25,6 +28,7 @@ import com.bivashy.backend.composer.model.TrackSource; import com.bivashy.backend.composer.model.TrackSourceMetadata; import com.bivashy.backend.composer.repository.TrackRepository; import com.bivashy.backend.composer.service.MetadataParseService.Metadata; +import com.bivashy.backend.composer.service.importing.RedisProgressService; import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; import com.fasterxml.jackson.databind.ObjectMapper; import com.jfposton.ytdlp.YtDlp; @@ -46,16 +50,19 @@ public class YoutubeTrackService { private final TrackMetadataService trackMetadataService; private final TrackPlaylistService trackPlaylistService; private final TrackSourceService trackSourceService; + private final RedisProgressService redisProgressService; public YoutubeTrackService(AudioS3StorageService s3StorageService, MetadataParseService metadataParseService, TrackRepository trackRepository, TrackMetadataService trackMetadataService, - TrackPlaylistService trackPlaylistService, TrackSourceService trackSourceService) { + TrackPlaylistService trackPlaylistService, TrackSourceService trackSourceService, + RedisProgressService redisProgressService) { this.s3StorageService = s3StorageService; this.metadataParseService = metadataParseService; this.trackRepository = trackRepository; this.trackMetadataService = trackMetadataService; this.trackPlaylistService = trackPlaylistService; this.trackSourceService = trackSourceService; + this.redisProgressService = redisProgressService; } public AddLocalTrackParams downloadYoutubeTrack(Path temporaryFolder, VideoInfo videoInfo, String youtubeUrl) @@ -85,7 +92,8 @@ public class YoutubeTrackService { throw new ImportTrackException("cannot download any youtube track"); } - public List refreshYoutubePlaylist(long playlistId, long sourceId) throws ImportTrackException { + public List refreshYoutubePlaylist(CustomUserDetails user, long playlistId, long sourceId) + throws ImportTrackException { Optional trackSourceMetadataOpt = trackSourceService.findWithMetadata(sourceId); if (trackSourceMetadataOpt.isEmpty()) throw new ImportTrackException("cannot find track source with metadata with id " + sourceId); @@ -98,10 +106,11 @@ public class YoutubeTrackService { } catch (YtDlpException e) { throw new ImportTrackException("cannot `yt-dlp --dump-json` from " + youtubeUrl, e); } - return refreshYoutubePlaylist(playlistId, trackSourceMetadata.getSource(), videoInfos, youtubeUrl); + return refreshYoutubePlaylist(user.getId(), playlistId, trackSourceMetadata.getSource(), videoInfos, + youtubeUrl); } - public List refreshYoutubePlaylist(long playlistId, TrackSource trackSource, + public List refreshYoutubePlaylist(long userId, long playlistId, TrackSource trackSource, List videoInfos, String youtubeUrl) throws ImportTrackException { List result = new ArrayList<>(); @@ -126,10 +135,29 @@ public class YoutubeTrackService { ytDlpRequest.setOption("audio-quality", 0); ytDlpRequest.setOption("audio-format", "best"); ytDlpRequest.setOption("no-overwrites"); - var response = YtDlp.execute(ytDlpRequest); - logger.info("yt dlp response {}", response); - // TODO: write to RedisProgressService + PlaylistProgress playlistProgress = new PlaylistProgress(playlistId, trackSource.getId(), userId, + videoInfos.size()); + redisProgressService.saveProgress(playlistProgress); + + var response = YtDlp.execute(ytDlpRequest, (downloadProgress, ignored) -> { + redisProgressService.updateTrackProgressField(playlistId, trackSource.getId(), userId, + progress -> { + progress.setOverallProgress((int) downloadProgress); + }); + + }, stdoutLine -> { + redisProgressService.updateTrackProgressField(playlistId, trackSource.getId(), userId, + progress -> { + progress.setYtdlnStdout(String.join("\n", progress.getYtdlnStdout(), stdoutLine)); + }); + }, null); + redisProgressService.updateTrackProgressField(playlistId, trackSource.getId(), userId, + progress -> { + progress.setOverallProgress(100); + progress.setStatus(ProgressStatus.FINISHED); + }); + logger.info("yt dlp response {}", response); try (Stream pathStream = Files.walk(temporaryFolder)) { List downloadedFiles = Files.walk(temporaryFolder).toList(); diff --git a/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java b/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java index bfaeab8..e96d64e 100644 --- a/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java +++ b/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java @@ -1,28 +1,35 @@ package com.bivashy.backend.composer.service.importing; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.redis.connection.Message; import org.springframework.data.redis.connection.MessageListener; import org.springframework.data.redis.listener.ChannelTopic; import org.springframework.data.redis.listener.RedisMessageListenerContainer; import org.springframework.stereotype.Component; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; import com.bivashy.backend.composer.dto.importing.ImportTrackKey; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Consumer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; @Component public class RedisMessageSubscriber { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Logger logger = LoggerFactory.getLogger(Logger.class); private final RedisMessageListenerContainer container; - private final Map> subscriptions = new ConcurrentHashMap<>(); + private final Map> subscriptions = new ConcurrentHashMap<>(); public RedisMessageSubscriber(RedisMessageListenerContainer container) { this.container = container; } - public void subscribeToPlaylist(long playlistId, long userId, Consumer messageHandler) { + public void subscribeToPlaylist(long playlistId, long userId, Consumer messageHandler) { String channel = ImportTrackKey.redisChannelKey(playlistId, userId); String subscriptionKey = ImportTrackKey.subscriptionKey(playlistId, userId); @@ -32,7 +39,13 @@ public class RedisMessageSubscriber { public void onMessage(Message message, byte[] pattern) { String receivedMessage = new String(message.getBody()); if (subscriptions.containsKey(subscriptionKey)) { - messageHandler.accept(receivedMessage); + try { + BaseTrackProgress progress = OBJECT_MAPPER.readValue(receivedMessage, + BaseTrackProgress.class); + messageHandler.accept(progress); + } catch (JsonProcessingException e) { + logger.error("cannot deserialize message into BaseTrackProgress.class", e); + } } } }, new ChannelTopic(channel)); diff --git a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java index 1d9f4ad..8bc9851 100644 --- a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java +++ b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java @@ -1,14 +1,22 @@ package com.bivashy.backend.composer.service.importing; -import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; -import com.bivashy.backend.composer.dto.importing.ImportTrackKey; -import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; +import com.bivashy.backend.composer.dto.importing.ImportTrackKey; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; @Service public class RedisProgressService { @@ -27,11 +35,11 @@ public class RedisProgressService { String key = ImportTrackKey.progressKey(progress.getPlaylistId(), progress.getUserId()); String trackKey = ImportTrackKey.trackKey( progress.getPlaylistId(), - progress.getTrackId(), + progress.getTrackSourceId(), progress.getUserId()); String progressJson = objectMapper.writeValueAsString(progress); - redisTemplate.opsForHash().put(key, Long.toString(progress.getTrackId()), progressJson); + redisTemplate.opsForHash().put(key, Long.toString(progress.getTrackSourceId()), progressJson); redisTemplate.opsForValue().set(trackKey, progressJson); @@ -44,6 +52,38 @@ public class RedisProgressService { } } + public void updateTrackProgressField(long playlistId, long trackSourceId, long userId, + Consumer updater) { + try { + String trackKey = ImportTrackKey.trackKey(playlistId, trackSourceId, userId); + String hashKey = ImportTrackKey.progressKey(playlistId, userId); + + String existingJson = redisTemplate.opsForValue().get(trackKey); + if (existingJson == null) { + throw new RuntimeException("Track progress not found"); + } + + JavaType progressType = objectMapper.getTypeFactory() + .constructType(BaseTrackProgress.class); + + T progress = objectMapper.readValue(existingJson, progressType); + + updater.accept(progress); + + String updatedJson = objectMapper.writeValueAsString(progress); + redisTemplate.opsForHash().put(hashKey, Long.toString(trackSourceId), updatedJson); + redisTemplate.opsForValue().set(trackKey, updatedJson); + + redisTemplate.expire(hashKey, 24, TimeUnit.HOURS); + redisTemplate.expire(trackKey, 24, TimeUnit.HOURS); + + publishProgressUpdate(progress); + + } catch (Exception e) { + throw new RuntimeException("Failed to update track progress", e); + } + } + public List getPlaylistProgress(long playlistId, long userId) { try { String key = ImportTrackKey.progressKey(playlistId, userId); @@ -65,9 +105,9 @@ public class RedisProgressService { } } - public BaseTrackProgress getTrackProgress(long playlistId, long trackId, long userId) { + public BaseTrackProgress getTrackProgress(long playlistId, long trackSourceId, long userId) { try { - String key = ImportTrackKey.trackKey(playlistId, trackId, userId); + String key = ImportTrackKey.trackKey(playlistId, trackSourceId, userId); String progressJson = redisTemplate.opsForValue().get(key); if (progressJson != null) { -- 2.49.0 From 84d881b5cebae08dac2560a5c014bd1bc9614e30 Mon Sep 17 00:00:00 2001 From: bivashy Date: Thu, 8 Jan 2026 03:55:12 +0500 Subject: [PATCH 5/5] Add unique id to `TrackProgress` --- .../composer/dto/importing/BaseTrackProgress.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java index e33b59e..95a1e44 100644 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java @@ -1,5 +1,7 @@ package com.bivashy.backend.composer.dto.importing; +import java.util.UUID; + import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -9,20 +11,26 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = SingleTrackProgress.class, name = "TRACK"), }) public abstract class BaseTrackProgress { + protected UUID id; protected long playlistId; protected long trackSourceId; protected long userId; protected long timestamp; - private String type; + private ProgressEntryType type; public BaseTrackProgress(long playlistId, long trackSourceId, long userId) { + this.id = UUID.randomUUID(); this.playlistId = playlistId; this.trackSourceId = trackSourceId; this.userId = userId; this.timestamp = System.currentTimeMillis(); } + public UUID getId() { + return id; + } + public Long getTimestamp() { return timestamp; } @@ -31,7 +39,7 @@ public abstract class BaseTrackProgress { return userId; } - public String getType() { + public ProgressEntryType getType() { return type; } @@ -44,7 +52,7 @@ public abstract class BaseTrackProgress { } protected void setType(ProgressEntryType type) { - this.type = type.name(); + this.type = type; } } -- 2.49.0