diff --git a/Dockerfile.dev b/Dockerfile.dev index 220f6bc..d6f4884 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -5,10 +5,23 @@ RUN apt-get update && \ curl \ vim \ git \ + unzip \ + python3.11 \ ca-certificates \ ffmpeg && \ rm -rf /var/lib/apt/lists/* +RUN ln -sf /usr/bin/python3.11 /usr/bin/python3 + +RUN curl -fsSL https://bun.sh/install > install.sh && \ + chmod +x install.sh && \ + ./install.sh && \ + rm install.sh + +RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \ + chmod a+rx /usr/local/bin/yt-dlp + + # Create non-root user RUN groupadd --gid 1000 spring-app && \ useradd --uid 1000 --gid spring-app --shell /bin/bash --create-home spring-app diff --git a/pom.xml b/pom.xml index 047fca2..ae9b992 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,8 @@ 3.2.3 2.8.5 2024.08.29 + 2.0.6 + 51 @@ -124,6 +126,17 @@ jaffree ${jaffree.version} + + io.github.bivashy + yt-dlp-java + ${yt-dlp-java.version} + + + io.soabase.record-builder + record-builder-core + ${record-builder.version} + provided + org.postgresql @@ -160,6 +173,19 @@ org.springframework.boot spring-boot-maven-plugin + + org.apache.maven.plugins + maven-compiler-plugin + + + + io.soabase.record-builder + record-builder-processor + ${record-builder.version} + + + + diff --git a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java index 04508b9..c249ebb 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java @@ -1,6 +1,5 @@ package com.bivashy.backend.composer.controller; -import java.io.IOException; import java.util.List; import org.springframework.http.MediaType; @@ -14,40 +13,42 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; import com.bivashy.backend.composer.auth.CustomUserDetails; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; import com.bivashy.backend.composer.dto.track.AddLocalTrackRequest; +import com.bivashy.backend.composer.dto.track.AddYoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; -import com.bivashy.backend.composer.model.SourceTypes; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; import com.bivashy.backend.composer.service.TrackService; -import com.bivashy.backend.composer.service.importing.RedisProgressService; +import com.bivashy.backend.composer.util.SimpleBlob.MultipartBlob; @RestController public class TrackController { private final TrackService trackService; - private final RedisProgressService redisProgressService; - public TrackController(TrackService trackService, RedisProgressService redisProgressService) { + public TrackController(TrackService trackService) { this.trackService = trackService; - this.redisProgressService = redisProgressService; } @PostMapping(path = "/playlist/{playlistId}/track/local", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResponseEntity addLocalTrack( @AuthenticationPrincipal CustomUserDetails user, - @PathVariable Long playlistId, - @ModelAttribute AddLocalTrackRequest request) throws IOException { - TrackResponse response = trackService.addLocalTrack(user, playlistId, request); - redisProgressService.saveProgress(new TrackProgressDTO(playlistId, - response.trackId(), - response.title(), - response.fileFormat(), - SourceTypes.FILE, - 100, - null, - System.currentTimeMillis(), - user.getId())); + @PathVariable long playlistId, + @ModelAttribute AddLocalTrackRequest request) throws ImportTrackException { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new MultipartBlob(request.source())) + .build(); + TrackResponse response = trackService.addLocalTrack(user, playlistId, params); + return ResponseEntity.ok(response); + } + + @PostMapping(path = "/playlist/{playlistId}/track/youtube") + public ResponseEntity> addYoutubeTrack( + @AuthenticationPrincipal CustomUserDetails user, + @PathVariable long playlistId, + @RequestBody AddYoutubeTrackRequest request) throws ImportTrackException { + List response = trackService.addYoutubeTrack(user, playlistId, request); return ResponseEntity.ok(response); } diff --git a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java index 09838be..dfdec00 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java @@ -1,13 +1,19 @@ package com.bivashy.backend.composer.controller.importing; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + import org.springframework.http.MediaType; import org.springframework.http.codec.ServerSentEvent; import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; import com.bivashy.backend.composer.auth.CustomUserDetails; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; import com.bivashy.backend.composer.dto.importing.ImportTrackKey; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; import com.bivashy.backend.composer.service.importing.RedisMessageSubscriber; import com.bivashy.backend.composer.service.importing.RedisProgressService; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,10 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.http.HttpServletResponse; import reactor.core.publisher.Flux; import reactor.core.publisher.Sinks; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; @RestController public class ProgressSSEController { @@ -65,11 +67,12 @@ public class ProgressSSEController { .build()) .doFirst(() -> { try { - List existingProgresses = redisProgressService.getPlaylistProgress(playlistId, + List existingProgresses = redisProgressService.getPlaylistProgress( + playlistId, userId); ObjectMapper mapper = new ObjectMapper(); - for (TrackProgressDTO progress : existingProgresses) { + for (BaseTrackProgress progress : existingProgresses) { sink.tryEmitNext(mapper.writeValueAsString(progress)); } } catch (Exception e) { diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java new file mode 100644 index 0000000..da33855 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java @@ -0,0 +1,50 @@ +package com.bivashy.backend.composer.dto.importing; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = PlaylistProgress.class, name = "PLAYLIST"), + @JsonSubTypes.Type(value = SingleTrackProgress.class, name = "TRACK"), +}) +public abstract class BaseTrackProgress { + protected long playlistId; + protected long trackId; + protected long userId; + + protected long timestamp; + private String type; + + public BaseTrackProgress(long playlistId, long trackId, long userId) { + this.playlistId = playlistId; + this.trackId = trackId; + this.userId = userId; + this.timestamp = System.currentTimeMillis(); + } + + public Long getTimestamp() { + return timestamp; + } + + public Long getUserId() { + return userId; + } + + public String getType() { + return type; + } + + public long getPlaylistId() { + return playlistId; + } + + public long getTrackId() { + return trackId; + } + + protected void setType(ProgressEntryType type) { + this.type = type.name(); + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java new file mode 100644 index 0000000..73b09a7 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java @@ -0,0 +1,37 @@ +package com.bivashy.backend.composer.dto.importing; + +public class PlaylistProgress extends BaseTrackProgress { + private String ytdlnStdout; + private int overallProgress; + private String status; + + public PlaylistProgress(long playlistId, long trackId, long userId) { + super(playlistId, trackId, userId); + this.setType(ProgressEntryType.PLAYLIST); + this.status = "LOADING"; + } + + public String getYtdlnStdout() { + return ytdlnStdout; + } + + public void setYtdlnStdout(String ytdlnStdout) { + this.ytdlnStdout = ytdlnStdout; + } + + public int getOverallProgress() { + return overallProgress; + } + + public void setOverallProgress(int overallProgress) { + this.overallProgress = overallProgress; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java new file mode 100644 index 0000000..65a2b06 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java @@ -0,0 +1,7 @@ +package com.bivashy.backend.composer.dto.importing; + +public enum ProgressEntryType { + PLAYLIST, + TRACK, + EXTERNAL_TRACK +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java new file mode 100644 index 0000000..cd2c957 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java @@ -0,0 +1,21 @@ +package com.bivashy.backend.composer.dto.importing; + +public class SingleTrackProgress extends BaseTrackProgress { + private String title; + private String format; + + public SingleTrackProgress(long playlistId, long trackId, long userId, String title, String format) { + super(playlistId, trackId, userId); + this.setType(ProgressEntryType.TRACK); + this.title = title; + this.format = format; + } + + public String getTitle() { + return title; + } + + public String getFormat() { + return format; + } +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java b/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java deleted file mode 100644 index da4b2ad..0000000 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.bivashy.backend.composer.dto.importing; - -public class TrackProgressDTO { - private long playlistId; - private long trackId; - private String trackTitle; - private String format; - private String sourceType; - private int progress; - private String metadata; - private Long timestamp; - private long userId; - - public TrackProgressDTO() { - } - - public TrackProgressDTO(long playlistId, long trackId, long userId) { - this.playlistId = playlistId; - this.trackId = trackId; - this.userId = userId; - this.timestamp = System.currentTimeMillis(); - } - - public TrackProgressDTO(long playlistId, - long trackId, - String trackTitle, - String format, - String sourceType, - int progress, - String metadata, - Long timestamp, - long userId) { - this.playlistId = playlistId; - this.trackId = trackId; - this.trackTitle = trackTitle; - this.format = format; - this.sourceType = sourceType; - this.progress = progress; - this.metadata = metadata; - this.timestamp = timestamp; - this.userId = userId; - } - - public long getPlaylistId() { - return playlistId; - } - - public void setPlaylistId(long playlistId) { - this.playlistId = playlistId; - } - - public long getTrackId() { - return trackId; - } - - public void setTrackId(long trackId) { - this.trackId = trackId; - } - - public String getTrackTitle() { - return trackTitle; - } - - public void setTrackTitle(String trackTitle) { - this.trackTitle = trackTitle; - } - - public String getFormat() { - return format; - } - - public void setFormat(String format) { - this.format = format; - } - - public String getSourceType() { - return sourceType; - } - - public void setSourceType(String sourceType) { - this.sourceType = sourceType; - } - - public int getProgress() { - return progress; - } - - public void setProgress(int progress) { - this.progress = progress; - } - - public String getMetadata() { - return metadata; - } - - public void setMetadata(String metadata) { - this.metadata = metadata; - } - - public Long getTimestamp() { - return timestamp; - } - - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - public long getUserId() { - return userId; - } - - public void setUserId(long userId) { - this.userId = userId; - } - -} diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java b/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java new file mode 100644 index 0000000..bf68275 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/track/AddYoutubeTrackRequest.java @@ -0,0 +1,4 @@ +package com.bivashy.backend.composer.dto.track; + +public record AddYoutubeTrackRequest(String youtubeUrl) { +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java b/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java new file mode 100644 index 0000000..8f70d18 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java @@ -0,0 +1,9 @@ +package com.bivashy.backend.composer.dto.track.service; + +import com.bivashy.backend.composer.util.SimpleBlob; + +import io.soabase.recordbuilder.core.RecordBuilder; + +@RecordBuilder +public record AddLocalTrackParams(SimpleBlob blob, String ytdlpMetadata, boolean includeProgressHistory) { +} diff --git a/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java b/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java new file mode 100644 index 0000000..f5ba2a5 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java @@ -0,0 +1,11 @@ +package com.bivashy.backend.composer.exception; + +public class ImportTrackException extends Exception { + public ImportTrackException(String message, Exception cause) { + super(message, cause); + } + + public ImportTrackException(String message) { + super(message); + } +} diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java index 558506a..d9faaed 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java @@ -7,6 +7,8 @@ import java.util.Map; import org.springframework.http.MediaType; public interface AudioBlobStorageService { + String storeFolder(); + String store(InputStream inputStream); String store(byte[] data); @@ -15,6 +17,8 @@ public interface AudioBlobStorageService { String store(byte[] data, Map metadata); + String store(String key, byte[] data); + byte[] readRaw(String path) throws IOException; Blob read(String path); diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java index 1b96815..ee1a70b 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @Service @@ -36,6 +37,18 @@ public class AudioS3StorageService implements AudioBlobStorageService { this.tika = tika; } + @Override + public String storeFolder() { + String objectKey = newObjectName(); + if (!objectKey.endsWith("/")) + objectKey = objectKey + "/"; + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(), RequestBody.empty()); + return objectKey; + } + @Override public String store(InputStream inputStream) { return store(inputStream, Map.of()); @@ -61,7 +74,33 @@ public class AudioS3StorageService implements AudioBlobStorageService { .contentType(contentType) .metadata(metadata) .build(), RequestBody.fromBytes(data)); - return String.join("/", bucket, objectKey); + return objectKey; + } + + @Override + public String store(String key, byte[] data) { + try { + ResponseInputStream response = s3Client.getObject(GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .build()); + + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .contentType(response.response().contentType()) + .metadata(response.response().metadata()) + .build(), RequestBody.fromBytes(data)); + + return key; + } catch (NoSuchKeyException e) { + System.out.println("no existing found"); + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .build(), RequestBody.fromBytes(data)); + return key; + } } @Override @@ -71,9 +110,6 @@ public class AudioS3StorageService implements AudioBlobStorageService { @Override public Blob read(String path) { - if (path.startsWith(bucket + "/")) { - path = path.substring(bucket.length()); - } ResponseInputStream response = s3Client.getObject(GetObjectRequest.builder() .bucket(bucket) .key(path) diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackService.java b/src/main/java/com/bivashy/backend/composer/service/TrackService.java index c0f9471..40831ea 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackService.java @@ -1,56 +1,100 @@ package com.bivashy.backend.composer.service; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.server.ResponseStatusException; import com.bivashy.backend.composer.auth.CustomUserDetails; -import com.bivashy.backend.composer.dto.track.AddLocalTrackRequest; +import com.bivashy.backend.composer.dto.importing.SingleTrackProgress; +import com.bivashy.backend.composer.dto.track.AddYoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; import com.bivashy.backend.composer.model.SourceTypes; import com.bivashy.backend.composer.model.Track; import com.bivashy.backend.composer.model.TrackMetadata; import com.bivashy.backend.composer.model.TrackSource; import com.bivashy.backend.composer.repository.TrackRepository; import com.bivashy.backend.composer.service.MetadataParseService.Metadata; +import com.bivashy.backend.composer.service.importing.RedisProgressService; +import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jfposton.ytdlp.YtDlp; +import com.jfposton.ytdlp.YtDlpException; +import com.jfposton.ytdlp.YtDlpRequest; +import com.jfposton.ytdlp.mapper.VideoInfo; + +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; @Service public class TrackService { + private static final Logger logger = LoggerFactory.getLogger(TrackService.class); + public static final String DOWNLOADED_METADATA_FILE = "downloaded"; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final TrackRepository trackRepository; private final TrackSourceService trackSourceService; private final TrackMetadataService trackMetadataService; private final TrackPlaylistService trackPlaylistService; private final MetadataParseService metadataParseService; + private final RedisProgressService redisProgressService; + private final AudioS3StorageService s3StorageService; - public TrackService(TrackRepository trackRepository, - TrackSourceService trackSourceService, - TrackMetadataService trackMetadataService, - TrackPlaylistService trackPlaylistService, - MetadataParseService metadataParseService) { + public TrackService(TrackRepository trackRepository, TrackSourceService trackSourceService, + TrackMetadataService trackMetadataService, TrackPlaylistService trackPlaylistService, + MetadataParseService metadataParseService, RedisProgressService redisProgressService, + AudioS3StorageService s3StorageService) { this.trackRepository = trackRepository; this.trackSourceService = trackSourceService; this.trackMetadataService = trackMetadataService; this.trackPlaylistService = trackPlaylistService; this.metadataParseService = metadataParseService; + this.redisProgressService = redisProgressService; + this.s3StorageService = s3StorageService; } - public TrackResponse addLocalTrack(CustomUserDetails user, Long playlistId, AddLocalTrackRequest request) - throws IOException { - Optional metadata = metadataParseService.extractMetadata(request.source().getInputStream()); + public TrackResponse addLocalTrack(CustomUserDetails user, + long playlistId, + AddLocalTrackParams params) + throws ImportTrackException { + var request = params.blob(); + Optional metadata = Optional.empty(); + try (var inputStream = request.inputStream()) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + request.fileName()); + } String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); - TrackSource trackSource = trackSourceService.createTrackSource( - request.source().getBytes(), ffprobeJson, SourceTypes.FILE); + TrackSource trackSource; + try { + trackSource = trackSourceService.createLocalTrackSource( + request.body(), ffprobeJson, params.ytdlpMetadata(), SourceTypes.FILE); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } Track track = trackRepository.save(new Track(trackSource)); - String fileName = fileNameWithoutExtension(request.source().getOriginalFilename()); + String fileName = fileNameWithoutExtension(request.fileName()); String title = metadata.map(Metadata::title).orElse(fileName); String artist = metadata.map(Metadata::artist).orElse(null); int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); @@ -66,6 +110,12 @@ public class TrackService { if (metadata.isPresent()) { fileFormat = metadata.map(m -> m.formatName()).get(); } + + if (params.includeProgressHistory()) { + redisProgressService + .saveProgress(new SingleTrackProgress(playlistId, track.getId(), user.getId(), title, fileFormat)); + } + return new TrackResponse( track.getId(), title, @@ -76,6 +126,174 @@ public class TrackService { fileName); } + @Transactional + public List addYoutubeTrack(CustomUserDetails user, long playlistId, + AddYoutubeTrackRequest request) throws ImportTrackException { + List videoInfos = Collections.emptyList(); + try { + videoInfos = YtDlp.getVideoInfo(request.youtubeUrl()); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot `yt-dlp --dump-json` from " + request.youtubeUrl(), e); + } + + logger.info("videoinfos count {}", videoInfos.size()); + + if (videoInfos.size() == 0) { + throw new ImportTrackException("cannot find videoInfos"); + } + + if (videoInfos.size() == 1) { + try { + VideoInfo videoInfo = videoInfos.get(0); + + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + var ytDlpRequest = new YtDlpRequest(request.youtubeUrl(), temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + var response = YtDlp.execute(ytDlpRequest); + // TODO: write to RedisProgressService + + TrackResponse result = null; + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + + if (downloadedFiles.isEmpty()) + throw new ImportTrackException("yt-dlp didn't downloaded anything for " + request.youtubeUrl()); + + for (Path downloadedFile : downloadedFiles) { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new PathBlob(downloadedFile)) + .ytdlpMetadata(OBJECT_MAPPER.writeValueAsString(videoInfo)) + .includeProgressHistory(false) + .build(); + + result = addLocalTrack(user, + playlistId, + params); + Files.delete(downloadedFile); + } + } + Files.delete(temporaryFolder); + return List.of(result); + } catch (IOException e) { + throw new ImportTrackException("i/o during single youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + request.youtubeUrl(), e); + } + } + + TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceTypes.PLAYLIST); + return refreshYoutubePlaylist(playlistId, trackSource, videoInfos, request.youtubeUrl()); + } + + public List refreshYoutubePlaylist(long playlistId, TrackSource trackSource, + List videoInfos, + String youtubeUrl) throws ImportTrackException { + List result = new ArrayList<>(); + logger.info(trackSource.getSourceUrl()); + try { + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + logger.info("temporaryFolder created {}", temporaryFolder.toString()); + + String downloadedMetadataKey = trackSource.getSourceUrl() + DOWNLOADED_METADATA_FILE; + try { + var rawBody = s3StorageService + .readRaw(downloadedMetadataKey); + Files.write(temporaryFolder.resolve(DOWNLOADED_METADATA_FILE), rawBody); + } catch (NoSuchKeyException e) { + logger.warn(".downloaded metadata file was not found, ignoring"); + } + + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + ytDlpRequest.setOption("download-archive", DOWNLOADED_METADATA_FILE); + ytDlpRequest.setOption("extract-audio"); + ytDlpRequest.setOption("audio-quality", 0); + ytDlpRequest.setOption("audio-format", "best"); + ytDlpRequest.setOption("no-overwrites"); + var response = YtDlp.execute(ytDlpRequest); + logger.info("yt dlp response {}", response); + + // TODO: write to RedisProgressService + + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + logger.info("downloaded file count {}", downloadedFiles.size()); + + for (Path path : downloadedFiles) { + if (Files.isDirectory(path)) + continue; + boolean isMetadataFile = path.getFileName().toString().equals(DOWNLOADED_METADATA_FILE); + var body = Files.readAllBytes(path); + + if (isMetadataFile) { + s3StorageService.store(downloadedMetadataKey, body); + continue; + } + String fileName = fileNameWithoutExtension(path.getFileName().toString()); + VideoInfo videoInfo = videoInfos.stream() + .filter(v -> v.getId().equals(fileName)) + .findFirst() + .orElseThrow(); + + String audioKey = trackSource.getSourceUrl() + UUID.randomUUID().toString(); + + logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); + + audioKey = s3StorageService.store(audioKey, body); + Optional metadata = Optional.empty(); + + try (var inputStream = Files.newInputStream(path)) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + path.toString()); + } + String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); + + TrackSource playlistEntrySource; + try { + playlistEntrySource = trackSourceService.createLocalTrackSource( + body, ffprobeJson, OBJECT_MAPPER.writeValueAsString(videoInfo), SourceTypes.PLAYLIST); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } + + Track track = trackRepository.save(new Track(playlistEntrySource)); + + String title = videoInfo.getTitle(); + String artist = metadata.map(Metadata::artist).orElse(null); + int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); + // TODO: thumbnail + // TODO: Recognize music if the duration is less than five minutes + // (configurable), and if not, it is a playlist and should be marked as is + trackMetadataService.createTrackMetadata( + track, title, fileName, trackSource.getSourceUrl(), artist, null, durationSeconds); + + trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); + + String fileFormat = "unknown"; + if (metadata.isPresent()) { + fileFormat = metadata.map(m -> m.formatName()).get(); + } + + var trackResponse = new TrackResponse( + track.getId(), + title, + artist, + audioKey, + fileFormat, + durationSeconds, + fileName); + result.add(trackResponse); + } + } + return result; + } catch (IOException e) { + throw new ImportTrackException("i/o during playlist youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + youtubeUrl, e); + } + } + public List getPlaylistTracks(CustomUserDetails user, Long playlistId) { return trackPlaylistService.getPlaylistTracks(playlistId).stream() .map(pt -> { diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java index f9827e1..ad56bee 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java @@ -1,6 +1,7 @@ package com.bivashy.backend.composer.service; import java.time.LocalDateTime; +import java.util.HashMap; import java.util.Map; import org.springframework.stereotype.Service; @@ -12,6 +13,9 @@ import com.bivashy.backend.composer.repository.TrackSourceRepository; @Service public class TrackSourceService { + public static final String FFPROBE_METADATA_KEY = "ffprobe"; + public static final String YTDLP_METADATA_KEY = "ytdlp"; + private final TrackSourceRepository trackSourceRepository; private final SourceTypeRepository sourceTypeRepository; private final AudioBlobStorageService s3Service; @@ -24,10 +28,25 @@ public class TrackSourceService { this.s3Service = s3Service; } - public TrackSource createTrackSource(byte[] audioBytes, String ffprobeJson, String sourceType) { - String audioPath = s3Service.store(audioBytes, Map.of("ffprobe", ffprobeJson)); + public TrackSource createLocalTrackSource(byte[] audioBytes, + String ffprobeJson, + String ytdlpMetadata, + String sourceType) { + Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); + if (ytdlpMetadata != null) { + // TODO: Add tag or smth? + } + String audioPath = s3Service.store(audioBytes, metadata); + SourceType type = sourceTypeRepository.findByName(sourceType) .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); } + + public TrackSource createYoutubeTrackSource(String sourceType) { + String folderPath = s3Service.storeFolder(); + SourceType type = sourceTypeRepository.findByName(sourceType) + .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); + return trackSourceRepository.save(new TrackSource(folderPath, type, LocalDateTime.now())); + } } diff --git a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java index d3e70ab..1d9f4ad 100644 --- a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java +++ b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java @@ -1,7 +1,7 @@ package com.bivashy.backend.composer.service.importing; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; import com.bivashy.backend.composer.dto.importing.ImportTrackKey; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.data.redis.core.StringRedisTemplate; @@ -22,7 +22,7 @@ public class RedisProgressService { this.objectMapper = objectMapper; } - public void saveProgress(TrackProgressDTO progress) { + public void saveProgress(BaseTrackProgress progress) { try { String key = ImportTrackKey.progressKey(progress.getPlaylistId(), progress.getUserId()); String trackKey = ImportTrackKey.trackKey( @@ -44,16 +44,16 @@ public class RedisProgressService { } } - public List getPlaylistProgress(long playlistId, long userId) { + public List getPlaylistProgress(long playlistId, long userId) { try { String key = ImportTrackKey.progressKey(playlistId, userId); Map progressMap = redisTemplate.opsForHash().entries(key); - List progressList = new ArrayList<>(); + List progressList = new ArrayList<>(); for (Object value : progressMap.values()) { - TrackProgressDTO progress = objectMapper.readValue( + BaseTrackProgress progress = objectMapper.readValue( (String) value, - TrackProgressDTO.class); + BaseTrackProgress.class); progressList.add(progress); } @@ -65,13 +65,13 @@ public class RedisProgressService { } } - public TrackProgressDTO getTrackProgress(long playlistId, long trackId, long userId) { + public BaseTrackProgress getTrackProgress(long playlistId, long trackId, long userId) { try { String key = ImportTrackKey.trackKey(playlistId, trackId, userId); String progressJson = redisTemplate.opsForValue().get(key); if (progressJson != null) { - return objectMapper.readValue(progressJson, TrackProgressDTO.class); + return objectMapper.readValue(progressJson, BaseTrackProgress.class); } return null; } catch (Exception e) { @@ -79,7 +79,7 @@ public class RedisProgressService { } } - private void publishProgressUpdate(TrackProgressDTO progress) { + private void publishProgressUpdate(BaseTrackProgress progress) { try { String channel = ImportTrackKey.redisChannelKey(progress.getPlaylistId(), progress.getUserId()); String message = objectMapper.writeValueAsString(progress); diff --git a/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java b/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java new file mode 100644 index 0000000..4757cb0 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java @@ -0,0 +1,64 @@ +package com.bivashy.backend.composer.util; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.springframework.web.multipart.MultipartFile; + +public interface SimpleBlob { + InputStream inputStream() throws IOException; + + byte[] body() throws IOException; + + String fileName(); + + public static class MultipartBlob implements SimpleBlob { + private final MultipartFile multipartFile; + + public MultipartBlob(MultipartFile multipartFile) { + this.multipartFile = multipartFile; + } + + @Override + public InputStream inputStream() throws IOException { + return multipartFile.getInputStream(); + } + + @Override + public byte[] body() throws IOException { + return multipartFile.getBytes(); + } + + @Override + public String fileName() { + return multipartFile.getOriginalFilename(); + } + + } + + public static class PathBlob implements SimpleBlob { + private final Path path; + + public PathBlob(Path path) { + this.path = path; + } + + @Override + public InputStream inputStream() throws IOException { + return Files.newInputStream(path); + } + + @Override + public byte[] body() throws IOException { + return Files.readAllBytes(path); + } + + @Override + public String fileName() { + return path.getFileName().toString(); + } + + } +}