From 2baa79c3a99af76fba9f6223a7abacbf19b4f482 Mon Sep 17 00:00:00 2001 From: bivashy Date: Mon, 5 Jan 2026 00:40:35 +0500 Subject: [PATCH] Implement `refresh` endpoint --- .gitignore | 2 + .../composer/controller/TrackController.java | 9 + .../backend/composer/model/Playlist.java | 2 +- .../composer/model/SourceMetadataType.java | 5 + .../backend/composer/model/SourceType.java | 46 +--- .../backend/composer/model/SourceTypes.java | 8 - .../backend/composer/model/TrackSource.java | 14 +- .../composer/model/TrackSourceMetadata.java | 47 ++++ .../bivashy/backend/composer/model/User.java | 2 +- .../repository/SourceTypeRepository.java | 11 - .../TrackSourceMetadataRepository.java | 18 ++ .../composer/service/TrackService.java | 173 ++------------ .../composer/service/TrackSourceService.java | 37 +-- .../composer/service/YoutubeTrackService.java | 216 ++++++++++++++++++ .../migration/V1_10__create_base_tables.sql | 117 +++++----- .../db/migration/V1_20__insert_enums.sql | 9 - 16 files changed, 405 insertions(+), 311 deletions(-) create mode 100644 src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java delete mode 100644 src/main/java/com/bivashy/backend/composer/model/SourceTypes.java create mode 100644 src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java delete mode 100644 src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java create mode 100644 src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java create mode 100644 src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java delete mode 100644 src/main/resources/db/migration/V1_20__insert_enums.sql diff --git a/.gitignore b/.gitignore index b9fcd77..22d23a8 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ build/ .vscode/ .env + +.sqruff diff --git a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java index 582416f..83ee40b 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java @@ -43,6 +43,15 @@ public class TrackController { return ResponseEntity.ok(response); } + @PostMapping(path = "/playlist/{playlistId}/track/youtube/refresh/{sourceId}") + public ResponseEntity> addYoutubeTrack( + @AuthenticationPrincipal CustomUserDetails user, + @PathVariable long playlistId, + @PathVariable long sourceId) throws ImportTrackException { + List response = trackService.refreshYoutubePlaylist(user, playlistId, sourceId); + return ResponseEntity.ok(response); + } + @PostMapping(path = "/playlist/{playlistId}/track/youtube") public ResponseEntity> addYoutubeTrack( @AuthenticationPrincipal CustomUserDetails user, diff --git a/src/main/java/com/bivashy/backend/composer/model/Playlist.java b/src/main/java/com/bivashy/backend/composer/model/Playlist.java index 061e0fc..ac95c60 100644 --- a/src/main/java/com/bivashy/backend/composer/model/Playlist.java +++ b/src/main/java/com/bivashy/backend/composer/model/Playlist.java @@ -28,7 +28,7 @@ public class Playlist { @JoinColumn(name = "owner_id", nullable = false) private User owner; - @Column(unique = true, nullable = false, length = 500) + @Column(unique = true, nullable = false) private String title; @Column(name = "created_at", nullable = false, updatable = false) diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java b/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java new file mode 100644 index 0000000..ae1c130 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java @@ -0,0 +1,5 @@ +package com.bivashy.backend.composer.model; + +public enum SourceMetadataType { + YOUTUBE +} diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceType.java b/src/main/java/com/bivashy/backend/composer/model/SourceType.java index d456aba..49d6d0b 100644 --- a/src/main/java/com/bivashy/backend/composer/model/SourceType.java +++ b/src/main/java/com/bivashy/backend/composer/model/SourceType.java @@ -1,47 +1,5 @@ package com.bivashy.backend.composer.model; -import java.util.HashSet; -import java.util.Set; - -import jakarta.persistence.CascadeType; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.OneToMany; -import jakarta.persistence.Table; - -@Entity -@Table(name = "source_type") -public class SourceType { - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column(nullable = false, length = 500) - private String name; - - @OneToMany(mappedBy = "sourceType", cascade = CascadeType.ALL, orphanRemoval = true) - private Set trackSources = new HashSet<>(); - - SourceType() { - } - - public SourceType(String name) { - this.name = name; - } - - public Long getId() { - return id; - } - - public String getName() { - return name; - } - - public Set getTrackSources() { - return trackSources; - } - +public enum SourceType { + VIDEO, PLAYLIST, PLAYLIST_ITEM, FILE, URL } diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java b/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java deleted file mode 100644 index f416568..0000000 --- a/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.bivashy.backend.composer.model; - -public class SourceTypes { - public static final String AUDIO = "VIDEO"; - public static final String PLAYLIST = "PLAYLIST"; - public static final String FILE = "FILE"; - public static final String URL = "URL"; -} diff --git a/src/main/java/com/bivashy/backend/composer/model/TrackSource.java b/src/main/java/com/bivashy/backend/composer/model/TrackSource.java index 2628303..7984f07 100644 --- a/src/main/java/com/bivashy/backend/composer/model/TrackSource.java +++ b/src/main/java/com/bivashy/backend/composer/model/TrackSource.java @@ -4,15 +4,16 @@ import java.time.LocalDateTime; import java.util.HashSet; import java.util.Set; +import org.hibernate.annotations.JdbcType; +import org.hibernate.dialect.PostgreSQLEnumJdbcType; + import jakarta.persistence.CascadeType; import jakarta.persistence.Column; import jakarta.persistence.Entity; -import jakarta.persistence.FetchType; +import jakarta.persistence.Enumerated; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; import jakarta.persistence.OneToMany; import jakarta.persistence.Table; @@ -23,11 +24,12 @@ public class TrackSource { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(name = "source_url", nullable = false, length = 500) + @Column(name = "source_url", nullable = false) private String sourceUrl; - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "source_type_id", nullable = false) + @Enumerated + @Column(name = "source_type", nullable = false) + @JdbcType(PostgreSQLEnumJdbcType.class) private SourceType sourceType; @Column(name = "last_fetched_at", nullable = false) diff --git a/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java b/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java new file mode 100644 index 0000000..4625f9e --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java @@ -0,0 +1,47 @@ +package com.bivashy.backend.composer.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; + +@Entity +@Table(name = "track_source_metadata") +public class TrackSourceMetadata { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "source_id", nullable = false, unique = true) + private TrackSource source; + + @Column(name = "url", nullable = false) + private String url; + + TrackSourceMetadata() { + } + + public TrackSourceMetadata(TrackSource source, String url) { + this.source = source; + this.url = url; + } + + public Long getId() { + return id; + } + + public TrackSource getSource() { + return source; + } + + public String getUrl() { + return url; + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/model/User.java b/src/main/java/com/bivashy/backend/composer/model/User.java index f0a049f..5241669 100644 --- a/src/main/java/com/bivashy/backend/composer/model/User.java +++ b/src/main/java/com/bivashy/backend/composer/model/User.java @@ -20,7 +20,7 @@ public class User { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(nullable = false, length = 500) + @Column(nullable = false) private String name; @Column(name = "created_at", nullable = false, updatable = false) diff --git a/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java b/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java deleted file mode 100644 index d73d4de..0000000 --- a/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.bivashy.backend.composer.repository; - -import java.util.Optional; - -import org.springframework.data.jpa.repository.JpaRepository; - -import com.bivashy.backend.composer.model.SourceType; - -public interface SourceTypeRepository extends JpaRepository { - Optional findByName(String name); -} diff --git a/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java b/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java new file mode 100644 index 0000000..95bd054 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java @@ -0,0 +1,18 @@ +package com.bivashy.backend.composer.repository; + +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.bivashy.backend.composer.model.TrackSourceMetadata; + +@Repository +public interface TrackSourceMetadataRepository extends JpaRepository { + @Query("SELECT tsm FROM TrackSourceMetadata tsm " + + "JOIN FETCH tsm.source " + + "WHERE tsm.source.id = :sourceId") + Optional findBySourceIdWithSource(@Param("sourceId") Long sourceId); +} diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackService.java b/src/main/java/com/bivashy/backend/composer/service/TrackService.java index 855474b..5aa48f2 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackService.java @@ -1,14 +1,13 @@ package com.bivashy.backend.composer.service; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.UUID; import java.util.stream.Stream; import org.slf4j.Logger; @@ -25,50 +24,42 @@ import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; import com.bivashy.backend.composer.dto.track.YoutubeTrackRequest; import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; -import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; import com.bivashy.backend.composer.exception.ImportTrackException; -import com.bivashy.backend.composer.model.SourceTypes; +import com.bivashy.backend.composer.model.SourceType; import com.bivashy.backend.composer.model.Track; import com.bivashy.backend.composer.model.TrackMetadata; import com.bivashy.backend.composer.model.TrackSource; import com.bivashy.backend.composer.repository.TrackRepository; import com.bivashy.backend.composer.service.MetadataParseService.Metadata; import com.bivashy.backend.composer.service.importing.RedisProgressService; -import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; -import com.fasterxml.jackson.databind.ObjectMapper; import com.jfposton.ytdlp.YtDlp; import com.jfposton.ytdlp.YtDlpException; -import com.jfposton.ytdlp.YtDlpRequest; import com.jfposton.ytdlp.mapper.VideoInfo; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; - @Service public class TrackService { private static final Logger logger = LoggerFactory.getLogger(TrackService.class); public static final String DOWNLOADED_METADATA_FILE = "downloaded"; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final TrackRepository trackRepository; private final TrackSourceService trackSourceService; private final TrackMetadataService trackMetadataService; private final TrackPlaylistService trackPlaylistService; private final MetadataParseService metadataParseService; private final RedisProgressService redisProgressService; - private final AudioS3StorageService s3StorageService; + private final YoutubeTrackService youtubeTrackService; public TrackService(TrackRepository trackRepository, TrackSourceService trackSourceService, TrackMetadataService trackMetadataService, TrackPlaylistService trackPlaylistService, MetadataParseService metadataParseService, RedisProgressService redisProgressService, - AudioS3StorageService s3StorageService) { + YoutubeTrackService youtubeTrackService) { this.trackRepository = trackRepository; this.trackSourceService = trackSourceService; this.trackMetadataService = trackMetadataService; this.trackPlaylistService = trackPlaylistService; this.metadataParseService = metadataParseService; this.redisProgressService = redisProgressService; - this.s3StorageService = s3StorageService; + this.youtubeTrackService = youtubeTrackService; } public TrackResponse addLocalTrack(CustomUserDetails user, @@ -87,7 +78,7 @@ public class TrackService { TrackSource trackSource; try { trackSource = trackSourceService.createLocalTrackSource( - request.body(), ffprobeJson, params.ytdlpMetadata(), SourceTypes.FILE); + request.body(), ffprobeJson, params.ytdlpMetadata(), SourceType.FILE); } catch (IOException e) { throw new ImportTrackException("cannot read blob body", e); } @@ -126,6 +117,12 @@ public class TrackService { fileName); } + @Transactional + public List refreshYoutubePlaylist(CustomUserDetails user, long playlistId, long sourceId) + throws ImportTrackException { + return youtubeTrackService.refreshYoutubePlaylist(playlistId, sourceId); + } + @Transactional public List addYoutubeTrack(CustomUserDetails user, long playlistId, YoutubeTrackRequest request) throws ImportTrackException { @@ -145,34 +142,17 @@ public class TrackService { if (videoInfos.size() == 1) { try { VideoInfo videoInfo = videoInfos.get(0); - Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); - var ytDlpRequest = new YtDlpRequest(request.youtubeUrl(), temporaryFolder.toAbsolutePath().toString()); - ytDlpRequest.setOption("output", "%(id)s"); - var response = YtDlp.execute(ytDlpRequest); - // TODO: write to RedisProgressService - TrackResponse result = null; + var params = youtubeTrackService.downloadYoutubeTrack(temporaryFolder, videoInfo, + request.youtubeUrl()); + TrackResponse result = addLocalTrack(user, playlistId, params); + try (Stream pathStream = Files.walk(temporaryFolder)) { - List downloadedFiles = Files.walk(temporaryFolder).toList(); - - if (downloadedFiles.isEmpty()) - throw new ImportTrackException("yt-dlp didn't downloaded anything for " + request.youtubeUrl()); - - for (Path downloadedFile : downloadedFiles) { - var params = AddLocalTrackParamsBuilder.builder() - .blob(new PathBlob(downloadedFile)) - .ytdlpMetadata(OBJECT_MAPPER.writeValueAsString(videoInfo)) - .includeProgressHistory(false) - .build(); - - result = addLocalTrack(user, - playlistId, - params); - Files.delete(downloadedFile); - } + pathStream.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); } - Files.delete(temporaryFolder); return List.of(result); } catch (IOException e) { throw new ImportTrackException("i/o during single youtube video downloading", e); @@ -181,116 +161,9 @@ public class TrackService { } } - TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceTypes.PLAYLIST); - return refreshYoutubePlaylist(playlistId, trackSource, videoInfos, request.youtubeUrl()); - } - - public List refreshYoutubePlaylist(long playlistId, TrackSource trackSource, - List videoInfos, - String youtubeUrl) throws ImportTrackException { - List result = new ArrayList<>(); - logger.info(trackSource.getSourceUrl()); - try { - Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); - logger.info("temporaryFolder created {}", temporaryFolder.toString()); - - String downloadedMetadataKey = trackSource.getSourceUrl() + DOWNLOADED_METADATA_FILE; - try { - var rawBody = s3StorageService - .readRaw(downloadedMetadataKey); - Files.write(temporaryFolder.resolve(DOWNLOADED_METADATA_FILE), rawBody); - } catch (NoSuchKeyException e) { - logger.warn(".downloaded metadata file was not found, ignoring"); - } - - var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); - ytDlpRequest.setOption("output", "%(id)s"); - ytDlpRequest.setOption("download-archive", DOWNLOADED_METADATA_FILE); - ytDlpRequest.setOption("extract-audio"); - ytDlpRequest.setOption("audio-quality", 0); - ytDlpRequest.setOption("audio-format", "best"); - ytDlpRequest.setOption("no-overwrites"); - var response = YtDlp.execute(ytDlpRequest); - logger.info("yt dlp response {}", response); - - // TODO: write to RedisProgressService - - try (Stream pathStream = Files.walk(temporaryFolder)) { - List downloadedFiles = Files.walk(temporaryFolder).toList(); - logger.info("downloaded file count {}", downloadedFiles.size()); - - for (Path path : downloadedFiles) { - if (Files.isDirectory(path)) - continue; - boolean isMetadataFile = path.getFileName().toString().equals(DOWNLOADED_METADATA_FILE); - var body = Files.readAllBytes(path); - - if (isMetadataFile) { - s3StorageService.store(downloadedMetadataKey, body, Map.of()); - continue; - } - String fileName = fileNameWithoutExtension(path.getFileName().toString()); - VideoInfo videoInfo = videoInfos.stream() - .filter(v -> v.getId().equals(fileName)) - .findFirst() - .orElseThrow(); - - String audioKey = trackSource.getSourceUrl() + UUID.randomUUID().toString(); - - logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); - - Optional metadata = Optional.empty(); - - try (var inputStream = Files.newInputStream(path)) { - metadata = metadataParseService.extractMetadata(inputStream); - } catch (IOException e) { - throw new ImportTrackException("cannot extract metadata from " + path.toString()); - } - String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); - - TrackSource playlistEntrySource; - try { - playlistEntrySource = trackSourceService.createTrackSourceWithKey(audioKey, body, ffprobeJson, - OBJECT_MAPPER.writeValueAsString(videoInfo), SourceTypes.PLAYLIST); - } catch (IOException e) { - throw new ImportTrackException("cannot read blob body", e); - } - - Track track = trackRepository.save(new Track(playlistEntrySource)); - - String title = videoInfo.getTitle(); - String artist = metadata.map(Metadata::artist).orElse(null); - int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); - // TODO: thumbnail - // TODO: Recognize music if the duration is less than five minutes - // (configurable), and if not, it is a playlist and should be marked as is - trackMetadataService.createTrackMetadata( - track, title, fileName, audioKey, artist, null, durationSeconds); - - trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); - - String fileFormat = "unknown"; - if (metadata.isPresent()) { - fileFormat = metadata.map(m -> m.formatName()).get(); - } - - var trackResponse = new TrackResponse( - track.getId(), - title, - artist, - audioKey, - fileFormat, - durationSeconds, - fileName); - result.add(trackResponse); - } - } - return result; - } catch (IOException e) { - throw new ImportTrackException("i/o during playlist youtube video downloading", e); - } catch (YtDlpException e) { - throw new ImportTrackException("cannot download youtube video " + youtubeUrl, e); - } + TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceType.PLAYLIST, + request.youtubeUrl()); + return youtubeTrackService.refreshYoutubePlaylist(playlistId, trackSource, videoInfos, request.youtubeUrl()); } public List getPlaylistTracks(CustomUserDetails user, Long playlistId) { diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java index eeebfff..2e45621 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java @@ -3,12 +3,14 @@ package com.bivashy.backend.composer.service; import java.time.LocalDateTime; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import org.springframework.stereotype.Service; import com.bivashy.backend.composer.model.SourceType; import com.bivashy.backend.composer.model.TrackSource; -import com.bivashy.backend.composer.repository.SourceTypeRepository; +import com.bivashy.backend.composer.model.TrackSourceMetadata; +import com.bivashy.backend.composer.repository.TrackSourceMetadataRepository; import com.bivashy.backend.composer.repository.TrackSourceRepository; @Service @@ -17,49 +19,50 @@ public class TrackSourceService { public static final String YTDLP_METADATA_KEY = "ytdlp"; private final TrackSourceRepository trackSourceRepository; - private final SourceTypeRepository sourceTypeRepository; + private final TrackSourceMetadataRepository trackSourceMetadataRepository; private final AudioBlobStorageService s3Service; public TrackSourceService(TrackSourceRepository trackSourceRepository, - SourceTypeRepository sourceTypeRepository, - AudioBlobStorageService s3Service) { + TrackSourceMetadataRepository trackSourceMetadataRepository, AudioBlobStorageService s3Service) { this.trackSourceRepository = trackSourceRepository; - this.sourceTypeRepository = sourceTypeRepository; + this.trackSourceMetadataRepository = trackSourceMetadataRepository; this.s3Service = s3Service; } public TrackSource createLocalTrackSource(byte[] audioBytes, String ffprobeJson, String ytdlpMetadata, - String sourceType) { + SourceType sourceType) { Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); if (ytdlpMetadata != null) { // TODO: Add tag or smth? } String audioPath = s3Service.store(audioBytes, metadata); - SourceType type = sourceTypeRepository.findByName(sourceType) - .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); - return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); + return trackSourceRepository.save(new TrackSource(audioPath, sourceType, LocalDateTime.now())); } public TrackSource createTrackSourceWithKey(String key, byte[] audioBytes, String ffprobeJson, - String ytdlpMetadata, String sourceType) { + String ytdlpMetadata, SourceType sourceType) { Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); if (ytdlpMetadata != null) { // TODO: Add tag or smth? } String audioPath = s3Service.store(key, audioBytes, metadata); - SourceType type = sourceTypeRepository.findByName(sourceType) - .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); - return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); + return trackSourceRepository.save(new TrackSource(audioPath, sourceType, LocalDateTime.now())); } - public TrackSource createYoutubeTrackSource(String sourceType) { + public TrackSource createYoutubeTrackSource(SourceType sourceType, String youtubeUrl) { String folderPath = s3Service.storeFolder(); - SourceType type = sourceTypeRepository.findByName(sourceType) - .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); - return trackSourceRepository.save(new TrackSource(folderPath, type, LocalDateTime.now())); + TrackSource trackSource = trackSourceRepository + .save(new TrackSource(folderPath, sourceType, LocalDateTime.now())); + trackSourceMetadataRepository.save(new TrackSourceMetadata(trackSource, youtubeUrl)); + return trackSource; } + + public Optional findWithMetadata(long sourceId) { + return trackSourceMetadataRepository.findBySourceIdWithSource(sourceId); + } + } diff --git a/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java new file mode 100644 index 0000000..ded3db1 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java @@ -0,0 +1,216 @@ +package com.bivashy.backend.composer.service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.bivashy.backend.composer.dto.track.TrackResponse; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; +import com.bivashy.backend.composer.model.SourceType; +import com.bivashy.backend.composer.model.Track; +import com.bivashy.backend.composer.model.TrackSource; +import com.bivashy.backend.composer.model.TrackSourceMetadata; +import com.bivashy.backend.composer.repository.TrackRepository; +import com.bivashy.backend.composer.service.MetadataParseService.Metadata; +import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jfposton.ytdlp.YtDlp; +import com.jfposton.ytdlp.YtDlpException; +import com.jfposton.ytdlp.YtDlpRequest; +import com.jfposton.ytdlp.mapper.VideoInfo; + +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; + +@Service +public class YoutubeTrackService { + private static final Logger logger = LoggerFactory.getLogger(YoutubeTrackService.class); + public static final String DOWNLOADED_METADATA_FILE = "downloaded"; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final AudioS3StorageService s3StorageService; + private final MetadataParseService metadataParseService; + private final TrackRepository trackRepository; + private final TrackMetadataService trackMetadataService; + private final TrackPlaylistService trackPlaylistService; + private final TrackSourceService trackSourceService; + + public YoutubeTrackService(AudioS3StorageService s3StorageService, MetadataParseService metadataParseService, + TrackRepository trackRepository, TrackMetadataService trackMetadataService, + TrackPlaylistService trackPlaylistService, TrackSourceService trackSourceService) { + this.s3StorageService = s3StorageService; + this.metadataParseService = metadataParseService; + this.trackRepository = trackRepository; + this.trackMetadataService = trackMetadataService; + this.trackPlaylistService = trackPlaylistService; + this.trackSourceService = trackSourceService; + } + + public AddLocalTrackParams downloadYoutubeTrack(Path temporaryFolder, VideoInfo videoInfo, String youtubeUrl) + throws IOException, YtDlpException, ImportTrackException { + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + var response = YtDlp.execute(ytDlpRequest); + // TODO: write to RedisProgressService + + TrackResponse result = null; + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + + if (downloadedFiles.isEmpty()) + throw new ImportTrackException("yt-dlp didn't downloaded anything for " + youtubeUrl); + + for (Path downloadedFile : downloadedFiles) { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new PathBlob(downloadedFile)) + .ytdlpMetadata(OBJECT_MAPPER.writeValueAsString(videoInfo)) + .includeProgressHistory(false) + .build(); + + return params; + } + } + throw new ImportTrackException("cannot download any youtube track"); + } + + public List refreshYoutubePlaylist(long playlistId, long sourceId) throws ImportTrackException { + Optional trackSourceMetadataOpt = trackSourceService.findWithMetadata(sourceId); + if (trackSourceMetadataOpt.isEmpty()) + throw new ImportTrackException("cannot find track source with metadata with id " + sourceId); + TrackSourceMetadata trackSourceMetadata = trackSourceMetadataOpt.get(); + String youtubeUrl = trackSourceMetadata.getUrl(); + + List videoInfos = Collections.emptyList(); + try { + videoInfos = YtDlp.getVideoInfo(youtubeUrl); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot `yt-dlp --dump-json` from " + youtubeUrl, e); + } + return refreshYoutubePlaylist(playlistId, trackSourceMetadata.getSource(), videoInfos, youtubeUrl); + } + + public List refreshYoutubePlaylist(long playlistId, TrackSource trackSource, + List videoInfos, + String youtubeUrl) throws ImportTrackException { + List result = new ArrayList<>(); + logger.info(trackSource.getSourceUrl()); + try { + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + logger.info("temporaryFolder created {}", temporaryFolder.toString()); + + String downloadedMetadataKey = trackSource.getSourceUrl() + DOWNLOADED_METADATA_FILE; + try { + var rawBody = s3StorageService + .readRaw(downloadedMetadataKey); + Files.write(temporaryFolder.resolve(DOWNLOADED_METADATA_FILE), rawBody); + } catch (NoSuchKeyException e) { + logger.warn(".downloaded metadata file was not found, ignoring"); + } + + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + ytDlpRequest.setOption("download-archive", DOWNLOADED_METADATA_FILE); + ytDlpRequest.setOption("extract-audio"); + ytDlpRequest.setOption("audio-quality", 0); + ytDlpRequest.setOption("audio-format", "best"); + ytDlpRequest.setOption("no-overwrites"); + var response = YtDlp.execute(ytDlpRequest); + logger.info("yt dlp response {}", response); + + // TODO: write to RedisProgressService + + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + logger.info("downloaded file count {}", downloadedFiles.size()); + + for (Path path : downloadedFiles) { + if (Files.isDirectory(path)) + continue; + boolean isMetadataFile = path.getFileName().toString().equals(DOWNLOADED_METADATA_FILE); + var body = Files.readAllBytes(path); + + if (isMetadataFile) { + s3StorageService.store(downloadedMetadataKey, body, Map.of()); + continue; + } + String fileName = fileNameWithoutExtension(path.getFileName().toString()); + VideoInfo videoInfo = videoInfos.stream() + .filter(v -> v.getId().equals(fileName)) + .findFirst() + .orElseThrow(); + + String audioKey = trackSource.getSourceUrl() + UUID.randomUUID().toString(); + + logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); + + Optional metadata = Optional.empty(); + + try (var inputStream = Files.newInputStream(path)) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + path.toString()); + } + String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); + + TrackSource playlistEntrySource; + try { + playlistEntrySource = trackSourceService.createTrackSourceWithKey(audioKey, body, ffprobeJson, + OBJECT_MAPPER.writeValueAsString(videoInfo), SourceType.PLAYLIST_ITEM); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } + + Track track = trackRepository.save(new Track(playlistEntrySource)); + + String title = videoInfo.getTitle(); + String artist = metadata.map(Metadata::artist).orElse(null); + int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); + // TODO: thumbnail + // TODO: Recognize music if the duration is less than five minutes + // (configurable), and if not, it is a playlist and should be marked as is + trackMetadataService.createTrackMetadata( + track, title, fileName, audioKey, artist, null, durationSeconds); + + trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); + + String fileFormat = "unknown"; + if (metadata.isPresent()) { + fileFormat = metadata.map(m -> m.formatName()).get(); + } + + var trackResponse = new TrackResponse( + track.getId(), + title, + artist, + audioKey, + fileFormat, + durationSeconds, + fileName); + result.add(trackResponse); + } + } + return result; + } catch (IOException e) { + throw new ImportTrackException("i/o during playlist youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + youtubeUrl, e); + } + } + + private String fileNameWithoutExtension(String fileName) { + return fileName.replaceFirst("[.][^.]+$", ""); + } + +} diff --git a/src/main/resources/db/migration/V1_10__create_base_tables.sql b/src/main/resources/db/migration/V1_10__create_base_tables.sql index 5c4c53b..262d2b7 100644 --- a/src/main/resources/db/migration/V1_10__create_base_tables.sql +++ b/src/main/resources/db/migration/V1_10__create_base_tables.sql @@ -1,80 +1,69 @@ -CREATE TABLE IF NOT EXISTS "users" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "name" varchar(500) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW() +CREATE TABLE IF NOT EXISTS users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE TABLE IF NOT EXISTS "source_type" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "name" varchar(500) NOT NULL +CREATE TYPE source_type_enum AS ENUM ( + 'VIDEO', 'PLAYLIST', 'PLAYLIST_ITEM', 'FILE', 'URL' ); -CREATE TABLE IF NOT EXISTS "track_source" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "source_url" varchar(500) NOT NULL, - "source_type_id" bigint NOT NULL, - "last_fetched_at" timestamp NOT NULL DEFAULT NOW(), - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_track_source_source_type_id" - FOREIGN KEY ("source_type_id") REFERENCES "source_type" ("id") +CREATE TABLE IF NOT EXISTS track_source ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_url TEXT NOT NULL, + source_type SOURCE_TYPE_ENUM NOT NULL, + last_fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE TABLE IF NOT EXISTS "track" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "source_id" bigint NOT NULL, - CONSTRAINT "fk_track_source_id" - FOREIGN KEY ("source_id") REFERENCES "track_source" ("id") +CREATE TABLE IF NOT EXISTS track_source_metadata ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_id BIGINT NOT NULL UNIQUE REFERENCES track_source ( + id + ) ON DELETE CASCADE, + url TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS "track_metadata" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "track_id" bigint NOT NULL, - "title" varchar(500) NOT NULL, - "file_name" varchar(500) NOT NULL, - "audio_path" varchar(500) NOT NULL, - "artist" varchar(500), - "thumbnail_path" varchar(500), - "duration_seconds" integer, - CONSTRAINT "fk_track_metadata_track_id" - FOREIGN KEY ("track_id") REFERENCES "track" ("id") +CREATE TABLE IF NOT EXISTS track ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES track_source (id) ON DELETE RESTRICT ); -CREATE TABLE IF NOT EXISTS "playlist" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "owner_id" bigint NOT NULL, - "title" varchar(500) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_playlist_owner_id" - FOREIGN KEY ("owner_id") REFERENCES "users" ("id"), - CONSTRAINT "uq_playlist_owner_title" - UNIQUE ("owner_id", "title") +CREATE TABLE IF NOT EXISTS track_metadata ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + title TEXT NOT NULL, + file_name TEXT NOT NULL, + audio_path TEXT NOT NULL, + artist TEXT, + thumbnail_path TEXT, + duration_seconds INTEGER ); -CREATE TABLE IF NOT EXISTS "playlist_track" ( - "playlist_id" bigint NOT NULL, - "track_id" bigint NOT NULL, - "order_index" numeric NOT NULL, - CONSTRAINT "pk_playlist_track_new" PRIMARY KEY ("playlist_id", "track_id"), - CONSTRAINT "fk_playlist_track_playlist_id_new" - FOREIGN KEY ("playlist_id") REFERENCES "playlist" ("id"), - CONSTRAINT "fk_playlist_track_track_id_new" - FOREIGN KEY ("track_id") REFERENCES "track" ("id") +CREATE TABLE IF NOT EXISTS playlist ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + owner_id BIGINT NOT NULL REFERENCES users (id) ON DELETE CASCADE, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (owner_id, title) ); -CREATE TABLE IF NOT EXISTS "track_version" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "track_id" bigint NOT NULL, - "metadata_id" bigint NOT NULL, - "source_id" bigint NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_track_version_track_id" - FOREIGN KEY ("track_id") REFERENCES "track" ("id"), - CONSTRAINT "fk_track_version_metadata_id" - FOREIGN KEY ("metadata_id") REFERENCES "track_metadata" ("id"), - CONSTRAINT "fk_track_version_source_id" - FOREIGN KEY ("source_id") REFERENCES "track_source" ("id") +CREATE TABLE IF NOT EXISTS playlist_track ( + playlist_id BIGINT NOT NULL REFERENCES playlist (id) ON DELETE CASCADE, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + order_index DECIMAL NOT NULL, + PRIMARY KEY (playlist_id, track_id) ); +CREATE TABLE IF NOT EXISTS track_version ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + metadata_id BIGINT NOT NULL REFERENCES track_metadata ( + id + ) ON DELETE CASCADE, + source_id BIGINT NOT NULL REFERENCES track_source (id) ON DELETE RESTRICT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/src/main/resources/db/migration/V1_20__insert_enums.sql b/src/main/resources/db/migration/V1_20__insert_enums.sql deleted file mode 100644 index 652794c..0000000 --- a/src/main/resources/db/migration/V1_20__insert_enums.sql +++ /dev/null @@ -1,9 +0,0 @@ -INSERT INTO "source_type" ("id", "name") -OVERRIDING SYSTEM VALUE -VALUES - (1, 'VIDEO'), - (2, 'PLAYLIST'), - (3, 'FILE'), - (4, 'URL') -ON CONFLICT ("id") DO NOTHING; -