diff --git a/.gitignore b/.gitignore index b9fcd77..22d23a8 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ build/ .vscode/ .env + +.sqruff diff --git a/Dockerfile.dev b/Dockerfile.dev index 220f6bc..d6f4884 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -5,10 +5,23 @@ RUN apt-get update && \ curl \ vim \ git \ + unzip \ + python3.11 \ ca-certificates \ ffmpeg && \ rm -rf /var/lib/apt/lists/* +RUN ln -sf /usr/bin/python3.11 /usr/bin/python3 + +RUN curl -fsSL https://bun.sh/install > install.sh && \ + chmod +x install.sh && \ + ./install.sh && \ + rm install.sh + +RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \ + chmod a+rx /usr/local/bin/yt-dlp + + # Create non-root user RUN groupadd --gid 1000 spring-app && \ useradd --uid 1000 --gid spring-app --shell /bin/bash --create-home spring-app diff --git a/pom.xml b/pom.xml index 047fca2..7e96b67 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,8 @@ 3.2.3 2.8.5 2024.08.29 + 2.0.7 + 51 @@ -119,11 +121,27 @@ springdoc-openapi-starter-webmvc-ui ${springdoc-openapi.version} + + org.springdoc + springdoc-openapi-starter-webflux-ui + ${springdoc-openapi.version} + com.github.kokorin.jaffree jaffree ${jaffree.version} + + io.github.bivashy + yt-dlp-java + ${yt-dlp-java.version} + + + io.soabase.record-builder + record-builder-core + ${record-builder.version} + provided + org.postgresql @@ -160,6 +178,19 @@ org.springframework.boot spring-boot-maven-plugin + + org.apache.maven.plugins + maven-compiler-plugin + + + + io.soabase.record-builder + record-builder-processor + ${record-builder.version} + + + + diff --git a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java index 04508b9..83ee40b 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/TrackController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/TrackController.java @@ -1,6 +1,5 @@ package com.bivashy.backend.composer.controller; -import java.io.IOException; import java.util.List; import org.springframework.http.MediaType; @@ -14,40 +13,51 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; import com.bivashy.backend.composer.auth.CustomUserDetails; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; import com.bivashy.backend.composer.dto.track.AddLocalTrackRequest; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; -import com.bivashy.backend.composer.model.SourceTypes; +import com.bivashy.backend.composer.dto.track.YoutubeTrackRequest; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; import com.bivashy.backend.composer.service.TrackService; -import com.bivashy.backend.composer.service.importing.RedisProgressService; +import com.bivashy.backend.composer.util.SimpleBlob.MultipartBlob; @RestController public class TrackController { private final TrackService trackService; - private final RedisProgressService redisProgressService; - public TrackController(TrackService trackService, RedisProgressService redisProgressService) { + public TrackController(TrackService trackService) { this.trackService = trackService; - this.redisProgressService = redisProgressService; } @PostMapping(path = "/playlist/{playlistId}/track/local", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResponseEntity addLocalTrack( @AuthenticationPrincipal CustomUserDetails user, - @PathVariable Long playlistId, - @ModelAttribute AddLocalTrackRequest request) throws IOException { - TrackResponse response = trackService.addLocalTrack(user, playlistId, request); - redisProgressService.saveProgress(new TrackProgressDTO(playlistId, - response.trackId(), - response.title(), - response.fileFormat(), - SourceTypes.FILE, - 100, - null, - System.currentTimeMillis(), - user.getId())); + @PathVariable long playlistId, + @ModelAttribute AddLocalTrackRequest request) throws ImportTrackException { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new MultipartBlob(request.source())) + .build(); + TrackResponse response = trackService.addLocalTrack(user, playlistId, params); + return ResponseEntity.ok(response); + } + + @PostMapping(path = "/playlist/{playlistId}/track/youtube/refresh/{sourceId}") + public ResponseEntity> addYoutubeTrack( + @AuthenticationPrincipal CustomUserDetails user, + @PathVariable long playlistId, + @PathVariable long sourceId) throws ImportTrackException { + List response = trackService.refreshYoutubePlaylist(user, playlistId, sourceId); + return ResponseEntity.ok(response); + } + + @PostMapping(path = "/playlist/{playlistId}/track/youtube") + public ResponseEntity> addYoutubeTrack( + @AuthenticationPrincipal CustomUserDetails user, + @PathVariable long playlistId, + @RequestBody YoutubeTrackRequest request) throws ImportTrackException { + List response = trackService.addYoutubeTrack(user, playlistId, request); return ResponseEntity.ok(response); } diff --git a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java index 09838be..b6d4b92 100644 --- a/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java +++ b/src/main/java/com/bivashy/backend/composer/controller/importing/ProgressSSEController.java @@ -1,31 +1,34 @@ package com.bivashy.backend.composer.controller.importing; -import org.springframework.http.MediaType; -import org.springframework.http.codec.ServerSentEvent; -import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.web.bind.annotation.*; - -import com.bivashy.backend.composer.auth.CustomUserDetails; -import com.bivashy.backend.composer.dto.importing.ImportTrackKey; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; -import com.bivashy.backend.composer.service.importing.RedisMessageSubscriber; -import com.bivashy.backend.composer.service.importing.RedisProgressService; -import com.fasterxml.jackson.databind.ObjectMapper; - -import jakarta.servlet.http.HttpServletResponse; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Sinks; -import java.time.Duration; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; + +import com.bivashy.backend.composer.auth.CustomUserDetails; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; +import com.bivashy.backend.composer.dto.importing.ImportTrackKey; +import com.bivashy.backend.composer.service.importing.RedisMessageSubscriber; +import com.bivashy.backend.composer.service.importing.RedisProgressService; + +import jakarta.servlet.http.HttpServletResponse; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Sinks; + @RestController public class ProgressSSEController { + private static final Logger logger = LoggerFactory.getLogger(ProgressSSEController.class); private final RedisProgressService redisProgressService; private final RedisMessageSubscriber redisSubscriber; - private final Map> sinks = new ConcurrentHashMap<>(); + private final Map> sinks = new ConcurrentHashMap<>(); public ProgressSSEController(RedisProgressService redisProgressService, RedisMessageSubscriber redisSubscriber) { @@ -34,7 +37,7 @@ public class ProgressSSEController { } @GetMapping(value = "/importing/stream/{playlistId}", produces = MediaType.TEXT_EVENT_STREAM_VALUE) - public Flux> streamProgress( + public Flux streamProgress( @PathVariable long playlistId, @AuthenticationPrincipal CustomUserDetails user, HttpServletResponse response) { @@ -46,8 +49,8 @@ public class ProgressSSEController { String connectionKey = ImportTrackKey.subscriptionKey(playlistId, userId); - Sinks.Many sink = sinks.computeIfAbsent(connectionKey, k -> { - Sinks.Many newSink = Sinks.many().replay().latest(); + Sinks.Many sink = sinks.computeIfAbsent(connectionKey, k -> { + Sinks.Many newSink = Sinks.many().replay().latest(); redisSubscriber.subscribeToPlaylist(playlistId, userId, message -> { newSink.tryEmitNext(message); @@ -59,18 +62,14 @@ public class ProgressSSEController { redisProgressService.addActiveConnection(playlistId, userId); return sink.asFlux() - .map(data -> ServerSentEvent.builder() - .data(data) - .event("progress-update") - .build()) .doFirst(() -> { try { - List existingProgresses = redisProgressService.getPlaylistProgress(playlistId, + List existingProgresses = redisProgressService.getPlaylistProgress( + playlistId, userId); - ObjectMapper mapper = new ObjectMapper(); - for (TrackProgressDTO progress : existingProgresses) { - sink.tryEmitNext(mapper.writeValueAsString(progress)); + for (BaseTrackProgress progress : existingProgresses) { + sink.tryEmitNext(progress); } } catch (Exception e) { e.printStackTrace(); @@ -89,7 +88,7 @@ public class ProgressSSEController { } private void cleanupConnection(Long playlistId, long userId, - Sinks.Many sink, String connectionKey) { + Sinks.Many sink, String connectionKey) { try { redisProgressService.removeActiveConnection(playlistId, userId); redisSubscriber.unsubscribeFromPlaylist(playlistId, userId); diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java new file mode 100644 index 0000000..95a1e44 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/BaseTrackProgress.java @@ -0,0 +1,58 @@ +package com.bivashy.backend.composer.dto.importing; + +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = PlaylistProgress.class, name = "PLAYLIST"), + @JsonSubTypes.Type(value = SingleTrackProgress.class, name = "TRACK"), +}) +public abstract class BaseTrackProgress { + protected UUID id; + protected long playlistId; + protected long trackSourceId; + protected long userId; + + protected long timestamp; + private ProgressEntryType type; + + public BaseTrackProgress(long playlistId, long trackSourceId, long userId) { + this.id = UUID.randomUUID(); + this.playlistId = playlistId; + this.trackSourceId = trackSourceId; + this.userId = userId; + this.timestamp = System.currentTimeMillis(); + } + + public UUID getId() { + return id; + } + + public Long getTimestamp() { + return timestamp; + } + + public Long getUserId() { + return userId; + } + + public ProgressEntryType getType() { + return type; + } + + public long getPlaylistId() { + return playlistId; + } + + public long getTrackSourceId() { + return trackSourceId; + } + + protected void setType(ProgressEntryType type) { + this.type = type; + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java b/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java index 6e4e4f2..6ac8ac8 100644 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/ImportTrackKey.java @@ -5,8 +5,8 @@ public class ImportTrackKey { return String.format("progress:%d:%d", userId, playlistId); } - public static String trackKey(long playlistId, long trackId, long userId) { - return String.format("track:%d:%d:%d", userId, playlistId, trackId); + public static String trackKey(long playlistId, long trackSourceId, long userId) { + return String.format("track:%d:%d:%d", userId, playlistId, trackSourceId); } public static String redisChannelKey(long playlistId, long userId) { diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java new file mode 100644 index 0000000..7d90293 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/PlaylistProgress.java @@ -0,0 +1,48 @@ +package com.bivashy.backend.composer.dto.importing; + +public class PlaylistProgress extends BaseTrackProgress { + private String ytdlnStdout; + private int overallProgress; + private int trackCount; + private ProgressStatus status; + + PlaylistProgress() { + super(0, 0, 0); + } + + public PlaylistProgress(long playlistId, long trackSourceId, long userId, int trackCount) { + super(playlistId, trackSourceId, userId); + this.setType(ProgressEntryType.PLAYLIST); + this.status = ProgressStatus.LOADING; + this.trackCount = trackCount; + } + + public String getYtdlnStdout() { + return ytdlnStdout; + } + + public void setYtdlnStdout(String ytdlnStdout) { + this.ytdlnStdout = ytdlnStdout; + } + + public int getOverallProgress() { + return overallProgress; + } + + public void setOverallProgress(int overallProgress) { + this.overallProgress = overallProgress; + } + + public ProgressStatus getStatus() { + return status; + } + + public void setStatus(ProgressStatus status) { + this.status = status; + } + + public int getTrackCount() { + return trackCount; + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java new file mode 100644 index 0000000..65a2b06 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressEntryType.java @@ -0,0 +1,7 @@ +package com.bivashy.backend.composer.dto.importing; + +public enum ProgressEntryType { + PLAYLIST, + TRACK, + EXTERNAL_TRACK +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressStatus.java b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressStatus.java new file mode 100644 index 0000000..68a79c1 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/ProgressStatus.java @@ -0,0 +1,5 @@ +package com.bivashy.backend.composer.dto.importing; + +public enum ProgressStatus { + LOADING, FINISHED +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java new file mode 100644 index 0000000..a98f018 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/importing/SingleTrackProgress.java @@ -0,0 +1,25 @@ +package com.bivashy.backend.composer.dto.importing; + +public class SingleTrackProgress extends BaseTrackProgress { + private String title; + private String format; + + SingleTrackProgress() { + super(0, 0, 0); + } + + public SingleTrackProgress(long playlistId, long trackSourceId, long userId, String title, String format) { + super(playlistId, trackSourceId, userId); + this.setType(ProgressEntryType.TRACK); + this.title = title; + this.format = format; + } + + public String getTitle() { + return title; + } + + public String getFormat() { + return format; + } +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java b/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java deleted file mode 100644 index da4b2ad..0000000 --- a/src/main/java/com/bivashy/backend/composer/dto/importing/TrackProgressDTO.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.bivashy.backend.composer.dto.importing; - -public class TrackProgressDTO { - private long playlistId; - private long trackId; - private String trackTitle; - private String format; - private String sourceType; - private int progress; - private String metadata; - private Long timestamp; - private long userId; - - public TrackProgressDTO() { - } - - public TrackProgressDTO(long playlistId, long trackId, long userId) { - this.playlistId = playlistId; - this.trackId = trackId; - this.userId = userId; - this.timestamp = System.currentTimeMillis(); - } - - public TrackProgressDTO(long playlistId, - long trackId, - String trackTitle, - String format, - String sourceType, - int progress, - String metadata, - Long timestamp, - long userId) { - this.playlistId = playlistId; - this.trackId = trackId; - this.trackTitle = trackTitle; - this.format = format; - this.sourceType = sourceType; - this.progress = progress; - this.metadata = metadata; - this.timestamp = timestamp; - this.userId = userId; - } - - public long getPlaylistId() { - return playlistId; - } - - public void setPlaylistId(long playlistId) { - this.playlistId = playlistId; - } - - public long getTrackId() { - return trackId; - } - - public void setTrackId(long trackId) { - this.trackId = trackId; - } - - public String getTrackTitle() { - return trackTitle; - } - - public void setTrackTitle(String trackTitle) { - this.trackTitle = trackTitle; - } - - public String getFormat() { - return format; - } - - public void setFormat(String format) { - this.format = format; - } - - public String getSourceType() { - return sourceType; - } - - public void setSourceType(String sourceType) { - this.sourceType = sourceType; - } - - public int getProgress() { - return progress; - } - - public void setProgress(int progress) { - this.progress = progress; - } - - public String getMetadata() { - return metadata; - } - - public void setMetadata(String metadata) { - this.metadata = metadata; - } - - public Long getTimestamp() { - return timestamp; - } - - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - public long getUserId() { - return userId; - } - - public void setUserId(long userId) { - this.userId = userId; - } - -} diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/YoutubeTrackRequest.java b/src/main/java/com/bivashy/backend/composer/dto/track/YoutubeTrackRequest.java new file mode 100644 index 0000000..0ae8ad5 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/track/YoutubeTrackRequest.java @@ -0,0 +1,4 @@ +package com.bivashy.backend.composer.dto.track; + +public record YoutubeTrackRequest(String youtubeUrl) { +} diff --git a/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java b/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java new file mode 100644 index 0000000..8f70d18 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/dto/track/service/AddLocalTrackParams.java @@ -0,0 +1,9 @@ +package com.bivashy.backend.composer.dto.track.service; + +import com.bivashy.backend.composer.util.SimpleBlob; + +import io.soabase.recordbuilder.core.RecordBuilder; + +@RecordBuilder +public record AddLocalTrackParams(SimpleBlob blob, String ytdlpMetadata, boolean includeProgressHistory) { +} diff --git a/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java b/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java new file mode 100644 index 0000000..f5ba2a5 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/exception/ImportTrackException.java @@ -0,0 +1,11 @@ +package com.bivashy.backend.composer.exception; + +public class ImportTrackException extends Exception { + public ImportTrackException(String message, Exception cause) { + super(message, cause); + } + + public ImportTrackException(String message) { + super(message); + } +} diff --git a/src/main/java/com/bivashy/backend/composer/model/Playlist.java b/src/main/java/com/bivashy/backend/composer/model/Playlist.java index 061e0fc..ac95c60 100644 --- a/src/main/java/com/bivashy/backend/composer/model/Playlist.java +++ b/src/main/java/com/bivashy/backend/composer/model/Playlist.java @@ -28,7 +28,7 @@ public class Playlist { @JoinColumn(name = "owner_id", nullable = false) private User owner; - @Column(unique = true, nullable = false, length = 500) + @Column(unique = true, nullable = false) private String title; @Column(name = "created_at", nullable = false, updatable = false) diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java b/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java new file mode 100644 index 0000000..ae1c130 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/model/SourceMetadataType.java @@ -0,0 +1,5 @@ +package com.bivashy.backend.composer.model; + +public enum SourceMetadataType { + YOUTUBE +} diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceType.java b/src/main/java/com/bivashy/backend/composer/model/SourceType.java index d456aba..49d6d0b 100644 --- a/src/main/java/com/bivashy/backend/composer/model/SourceType.java +++ b/src/main/java/com/bivashy/backend/composer/model/SourceType.java @@ -1,47 +1,5 @@ package com.bivashy.backend.composer.model; -import java.util.HashSet; -import java.util.Set; - -import jakarta.persistence.CascadeType; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.OneToMany; -import jakarta.persistence.Table; - -@Entity -@Table(name = "source_type") -public class SourceType { - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column(nullable = false, length = 500) - private String name; - - @OneToMany(mappedBy = "sourceType", cascade = CascadeType.ALL, orphanRemoval = true) - private Set trackSources = new HashSet<>(); - - SourceType() { - } - - public SourceType(String name) { - this.name = name; - } - - public Long getId() { - return id; - } - - public String getName() { - return name; - } - - public Set getTrackSources() { - return trackSources; - } - +public enum SourceType { + VIDEO, PLAYLIST, PLAYLIST_ITEM, FILE, URL } diff --git a/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java b/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java deleted file mode 100644 index f416568..0000000 --- a/src/main/java/com/bivashy/backend/composer/model/SourceTypes.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.bivashy.backend.composer.model; - -public class SourceTypes { - public static final String AUDIO = "VIDEO"; - public static final String PLAYLIST = "PLAYLIST"; - public static final String FILE = "FILE"; - public static final String URL = "URL"; -} diff --git a/src/main/java/com/bivashy/backend/composer/model/TrackSource.java b/src/main/java/com/bivashy/backend/composer/model/TrackSource.java index 2628303..7984f07 100644 --- a/src/main/java/com/bivashy/backend/composer/model/TrackSource.java +++ b/src/main/java/com/bivashy/backend/composer/model/TrackSource.java @@ -4,15 +4,16 @@ import java.time.LocalDateTime; import java.util.HashSet; import java.util.Set; +import org.hibernate.annotations.JdbcType; +import org.hibernate.dialect.PostgreSQLEnumJdbcType; + import jakarta.persistence.CascadeType; import jakarta.persistence.Column; import jakarta.persistence.Entity; -import jakarta.persistence.FetchType; +import jakarta.persistence.Enumerated; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; import jakarta.persistence.OneToMany; import jakarta.persistence.Table; @@ -23,11 +24,12 @@ public class TrackSource { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(name = "source_url", nullable = false, length = 500) + @Column(name = "source_url", nullable = false) private String sourceUrl; - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "source_type_id", nullable = false) + @Enumerated + @Column(name = "source_type", nullable = false) + @JdbcType(PostgreSQLEnumJdbcType.class) private SourceType sourceType; @Column(name = "last_fetched_at", nullable = false) diff --git a/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java b/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java new file mode 100644 index 0000000..4625f9e --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/model/TrackSourceMetadata.java @@ -0,0 +1,47 @@ +package com.bivashy.backend.composer.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; + +@Entity +@Table(name = "track_source_metadata") +public class TrackSourceMetadata { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "source_id", nullable = false, unique = true) + private TrackSource source; + + @Column(name = "url", nullable = false) + private String url; + + TrackSourceMetadata() { + } + + public TrackSourceMetadata(TrackSource source, String url) { + this.source = source; + this.url = url; + } + + public Long getId() { + return id; + } + + public TrackSource getSource() { + return source; + } + + public String getUrl() { + return url; + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/model/User.java b/src/main/java/com/bivashy/backend/composer/model/User.java index f0a049f..5241669 100644 --- a/src/main/java/com/bivashy/backend/composer/model/User.java +++ b/src/main/java/com/bivashy/backend/composer/model/User.java @@ -20,7 +20,7 @@ public class User { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(nullable = false, length = 500) + @Column(nullable = false) private String name; @Column(name = "created_at", nullable = false, updatable = false) diff --git a/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java b/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java deleted file mode 100644 index d73d4de..0000000 --- a/src/main/java/com/bivashy/backend/composer/repository/SourceTypeRepository.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.bivashy.backend.composer.repository; - -import java.util.Optional; - -import org.springframework.data.jpa.repository.JpaRepository; - -import com.bivashy.backend.composer.model.SourceType; - -public interface SourceTypeRepository extends JpaRepository { - Optional findByName(String name); -} diff --git a/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java b/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java new file mode 100644 index 0000000..95bd054 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/repository/TrackSourceMetadataRepository.java @@ -0,0 +1,18 @@ +package com.bivashy.backend.composer.repository; + +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.bivashy.backend.composer.model.TrackSourceMetadata; + +@Repository +public interface TrackSourceMetadataRepository extends JpaRepository { + @Query("SELECT tsm FROM TrackSourceMetadata tsm " + + "JOIN FETCH tsm.source " + + "WHERE tsm.source.id = :sourceId") + Optional findBySourceIdWithSource(@Param("sourceId") Long sourceId); +} diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java index 558506a..3b65e82 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioBlobStorageService.java @@ -7,6 +7,8 @@ import java.util.Map; import org.springframework.http.MediaType; public interface AudioBlobStorageService { + String storeFolder(); + String store(InputStream inputStream); String store(byte[] data); @@ -15,6 +17,8 @@ public interface AudioBlobStorageService { String store(byte[] data, Map metadata); + String store(String key, byte[] data, Map metadata); + byte[] readRaw(String path) throws IOException; Blob read(String path); diff --git a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java index 1b96815..5454518 100644 --- a/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java +++ b/src/main/java/com/bivashy/backend/composer/service/AudioS3StorageService.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @Service @@ -36,6 +37,18 @@ public class AudioS3StorageService implements AudioBlobStorageService { this.tika = tika; } + @Override + public String storeFolder() { + String objectKey = newObjectName(); + if (!objectKey.endsWith("/")) + objectKey = objectKey + "/"; + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(), RequestBody.empty()); + return objectKey; + } + @Override public String store(InputStream inputStream) { return store(inputStream, Map.of()); @@ -61,7 +74,34 @@ public class AudioS3StorageService implements AudioBlobStorageService { .contentType(contentType) .metadata(metadata) .build(), RequestBody.fromBytes(data)); - return String.join("/", bucket, objectKey); + return objectKey; + } + + @Override + public String store(String key, byte[] data, Map metadata) { + try { + ResponseInputStream response = s3Client.getObject(GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .build()); + + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .contentType(response.response().contentType()) + .metadata(metadata) + .build(), RequestBody.fromBytes(data)); + + return key; + } catch (NoSuchKeyException e) { + System.out.println("no existing found"); + s3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .metadata(metadata) + .build(), RequestBody.fromBytes(data)); + return key; + } } @Override @@ -71,9 +111,6 @@ public class AudioS3StorageService implements AudioBlobStorageService { @Override public Blob read(String path) { - if (path.startsWith(bucket + "/")) { - path = path.substring(bucket.length()); - } ResponseInputStream response = s3Client.getObject(GetObjectRequest.builder() .bucket(bucket) .key(path) diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackService.java b/src/main/java/com/bivashy/backend/composer/service/TrackService.java index c0f9471..3ab2fdd 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackService.java @@ -1,56 +1,91 @@ package com.bivashy.backend.composer.service; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.server.ResponseStatusException; import com.bivashy.backend.composer.auth.CustomUserDetails; -import com.bivashy.backend.composer.dto.track.AddLocalTrackRequest; +import com.bivashy.backend.composer.dto.importing.SingleTrackProgress; import com.bivashy.backend.composer.dto.track.PlaylistTrackResponse; import com.bivashy.backend.composer.dto.track.TrackBulkReorderRequest; import com.bivashy.backend.composer.dto.track.TrackResponse; -import com.bivashy.backend.composer.model.SourceTypes; +import com.bivashy.backend.composer.dto.track.YoutubeTrackRequest; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; +import com.bivashy.backend.composer.exception.ImportTrackException; +import com.bivashy.backend.composer.model.SourceType; import com.bivashy.backend.composer.model.Track; import com.bivashy.backend.composer.model.TrackMetadata; import com.bivashy.backend.composer.model.TrackSource; import com.bivashy.backend.composer.repository.TrackRepository; import com.bivashy.backend.composer.service.MetadataParseService.Metadata; +import com.bivashy.backend.composer.service.importing.RedisProgressService; +import com.jfposton.ytdlp.YtDlp; +import com.jfposton.ytdlp.YtDlpException; +import com.jfposton.ytdlp.mapper.VideoInfo; @Service public class TrackService { + private static final Logger logger = LoggerFactory.getLogger(TrackService.class); + public static final String DOWNLOADED_METADATA_FILE = "downloaded"; + private final TrackRepository trackRepository; private final TrackSourceService trackSourceService; private final TrackMetadataService trackMetadataService; private final TrackPlaylistService trackPlaylistService; private final MetadataParseService metadataParseService; + private final RedisProgressService redisProgressService; + private final YoutubeTrackService youtubeTrackService; - public TrackService(TrackRepository trackRepository, - TrackSourceService trackSourceService, - TrackMetadataService trackMetadataService, - TrackPlaylistService trackPlaylistService, - MetadataParseService metadataParseService) { + public TrackService(TrackRepository trackRepository, TrackSourceService trackSourceService, + TrackMetadataService trackMetadataService, TrackPlaylistService trackPlaylistService, + MetadataParseService metadataParseService, RedisProgressService redisProgressService, + YoutubeTrackService youtubeTrackService) { this.trackRepository = trackRepository; this.trackSourceService = trackSourceService; this.trackMetadataService = trackMetadataService; this.trackPlaylistService = trackPlaylistService; this.metadataParseService = metadataParseService; + this.redisProgressService = redisProgressService; + this.youtubeTrackService = youtubeTrackService; } - public TrackResponse addLocalTrack(CustomUserDetails user, Long playlistId, AddLocalTrackRequest request) - throws IOException { - Optional metadata = metadataParseService.extractMetadata(request.source().getInputStream()); + public TrackResponse addLocalTrack(CustomUserDetails user, + long playlistId, + AddLocalTrackParams params) + throws ImportTrackException { + var request = params.blob(); + Optional metadata = Optional.empty(); + try (var inputStream = request.inputStream()) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + request.fileName()); + } String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); - TrackSource trackSource = trackSourceService.createTrackSource( - request.source().getBytes(), ffprobeJson, SourceTypes.FILE); + TrackSource trackSource; + try { + trackSource = trackSourceService.createLocalTrackSource( + request.body(), ffprobeJson, params.ytdlpMetadata(), SourceType.FILE); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } Track track = trackRepository.save(new Track(trackSource)); - String fileName = fileNameWithoutExtension(request.source().getOriginalFilename()); + String fileName = fileNameWithoutExtension(request.fileName()); String title = metadata.map(Metadata::title).orElse(fileName); String artist = metadata.map(Metadata::artist).orElse(null); int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); @@ -66,6 +101,13 @@ public class TrackService { if (metadata.isPresent()) { fileFormat = metadata.map(m -> m.formatName()).get(); } + + if (params.includeProgressHistory()) { + redisProgressService + .saveProgress( + new SingleTrackProgress(playlistId, trackSource.getId(), user.getId(), title, fileFormat)); + } + return new TrackResponse( track.getId(), title, @@ -76,6 +118,56 @@ public class TrackService { fileName); } + @Transactional + public List refreshYoutubePlaylist(CustomUserDetails user, long playlistId, long sourceId) + throws ImportTrackException { + return youtubeTrackService.refreshYoutubePlaylist(user, playlistId, sourceId); + } + + @Transactional + public List addYoutubeTrack(CustomUserDetails user, long playlistId, + YoutubeTrackRequest request) throws ImportTrackException { + List videoInfos = Collections.emptyList(); + try { + videoInfos = YtDlp.getVideoInfo(request.youtubeUrl()); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot `yt-dlp --dump-json` from " + request.youtubeUrl(), e); + } + + logger.info("videoinfos count {}", videoInfos.size()); + + if (videoInfos.size() == 0) { + throw new ImportTrackException("cannot find videoInfos"); + } + + if (videoInfos.size() == 1) { + try { + VideoInfo videoInfo = videoInfos.get(0); + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + + var params = youtubeTrackService.downloadYoutubeTrack(temporaryFolder, videoInfo, + request.youtubeUrl()); + TrackResponse result = addLocalTrack(user, playlistId, params); + + try (Stream pathStream = Files.walk(temporaryFolder)) { + pathStream.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + return List.of(result); + } catch (IOException e) { + throw new ImportTrackException("i/o during single youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + request.youtubeUrl(), e); + } + } + + TrackSource trackSource = trackSourceService.createYoutubeTrackSource(SourceType.PLAYLIST, + request.youtubeUrl()); + return youtubeTrackService.refreshYoutubePlaylist(user.getId(), playlistId, trackSource, videoInfos, + request.youtubeUrl()); + } + public List getPlaylistTracks(CustomUserDetails user, Long playlistId) { return trackPlaylistService.getPlaylistTracks(playlistId).stream() .map(pt -> { diff --git a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java index f9827e1..2e45621 100644 --- a/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java +++ b/src/main/java/com/bivashy/backend/composer/service/TrackSourceService.java @@ -1,33 +1,68 @@ package com.bivashy.backend.composer.service; import java.time.LocalDateTime; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; import org.springframework.stereotype.Service; import com.bivashy.backend.composer.model.SourceType; import com.bivashy.backend.composer.model.TrackSource; -import com.bivashy.backend.composer.repository.SourceTypeRepository; +import com.bivashy.backend.composer.model.TrackSourceMetadata; +import com.bivashy.backend.composer.repository.TrackSourceMetadataRepository; import com.bivashy.backend.composer.repository.TrackSourceRepository; @Service public class TrackSourceService { + public static final String FFPROBE_METADATA_KEY = "ffprobe"; + public static final String YTDLP_METADATA_KEY = "ytdlp"; + private final TrackSourceRepository trackSourceRepository; - private final SourceTypeRepository sourceTypeRepository; + private final TrackSourceMetadataRepository trackSourceMetadataRepository; private final AudioBlobStorageService s3Service; public TrackSourceService(TrackSourceRepository trackSourceRepository, - SourceTypeRepository sourceTypeRepository, - AudioBlobStorageService s3Service) { + TrackSourceMetadataRepository trackSourceMetadataRepository, AudioBlobStorageService s3Service) { this.trackSourceRepository = trackSourceRepository; - this.sourceTypeRepository = sourceTypeRepository; + this.trackSourceMetadataRepository = trackSourceMetadataRepository; this.s3Service = s3Service; } - public TrackSource createTrackSource(byte[] audioBytes, String ffprobeJson, String sourceType) { - String audioPath = s3Service.store(audioBytes, Map.of("ffprobe", ffprobeJson)); - SourceType type = sourceTypeRepository.findByName(sourceType) - .orElseThrow(() -> new IllegalStateException("Source type not found: " + sourceType)); - return trackSourceRepository.save(new TrackSource(audioPath, type, LocalDateTime.now())); + public TrackSource createLocalTrackSource(byte[] audioBytes, + String ffprobeJson, + String ytdlpMetadata, + SourceType sourceType) { + Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); + if (ytdlpMetadata != null) { + // TODO: Add tag or smth? + } + String audioPath = s3Service.store(audioBytes, metadata); + + return trackSourceRepository.save(new TrackSource(audioPath, sourceType, LocalDateTime.now())); } + + public TrackSource createTrackSourceWithKey(String key, byte[] audioBytes, String ffprobeJson, + String ytdlpMetadata, SourceType sourceType) { + Map metadata = new HashMap<>(Map.of(YTDLP_METADATA_KEY, ffprobeJson)); + if (ytdlpMetadata != null) { + // TODO: Add tag or smth? + } + String audioPath = s3Service.store(key, audioBytes, metadata); + + return trackSourceRepository.save(new TrackSource(audioPath, sourceType, LocalDateTime.now())); + } + + public TrackSource createYoutubeTrackSource(SourceType sourceType, String youtubeUrl) { + String folderPath = s3Service.storeFolder(); + TrackSource trackSource = trackSourceRepository + .save(new TrackSource(folderPath, sourceType, LocalDateTime.now())); + trackSourceMetadataRepository.save(new TrackSourceMetadata(trackSource, youtubeUrl)); + return trackSource; + } + + public Optional findWithMetadata(long sourceId) { + return trackSourceMetadataRepository.findBySourceIdWithSource(sourceId); + } + } diff --git a/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java new file mode 100644 index 0000000..a6ae2e4 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/service/YoutubeTrackService.java @@ -0,0 +1,244 @@ +package com.bivashy.backend.composer.service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.bivashy.backend.composer.auth.CustomUserDetails; +import com.bivashy.backend.composer.dto.importing.PlaylistProgress; +import com.bivashy.backend.composer.dto.importing.ProgressStatus; +import com.bivashy.backend.composer.dto.track.TrackResponse; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParams; +import com.bivashy.backend.composer.dto.track.service.AddLocalTrackParamsBuilder; +import com.bivashy.backend.composer.exception.ImportTrackException; +import com.bivashy.backend.composer.model.SourceType; +import com.bivashy.backend.composer.model.Track; +import com.bivashy.backend.composer.model.TrackSource; +import com.bivashy.backend.composer.model.TrackSourceMetadata; +import com.bivashy.backend.composer.repository.TrackRepository; +import com.bivashy.backend.composer.service.MetadataParseService.Metadata; +import com.bivashy.backend.composer.service.importing.RedisProgressService; +import com.bivashy.backend.composer.util.SimpleBlob.PathBlob; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jfposton.ytdlp.YtDlp; +import com.jfposton.ytdlp.YtDlpException; +import com.jfposton.ytdlp.YtDlpRequest; +import com.jfposton.ytdlp.mapper.VideoInfo; + +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; + +@Service +public class YoutubeTrackService { + private static final Logger logger = LoggerFactory.getLogger(YoutubeTrackService.class); + public static final String DOWNLOADED_METADATA_FILE = "downloaded"; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final AudioS3StorageService s3StorageService; + private final MetadataParseService metadataParseService; + private final TrackRepository trackRepository; + private final TrackMetadataService trackMetadataService; + private final TrackPlaylistService trackPlaylistService; + private final TrackSourceService trackSourceService; + private final RedisProgressService redisProgressService; + + public YoutubeTrackService(AudioS3StorageService s3StorageService, MetadataParseService metadataParseService, + TrackRepository trackRepository, TrackMetadataService trackMetadataService, + TrackPlaylistService trackPlaylistService, TrackSourceService trackSourceService, + RedisProgressService redisProgressService) { + this.s3StorageService = s3StorageService; + this.metadataParseService = metadataParseService; + this.trackRepository = trackRepository; + this.trackMetadataService = trackMetadataService; + this.trackPlaylistService = trackPlaylistService; + this.trackSourceService = trackSourceService; + this.redisProgressService = redisProgressService; + } + + public AddLocalTrackParams downloadYoutubeTrack(Path temporaryFolder, VideoInfo videoInfo, String youtubeUrl) + throws IOException, YtDlpException, ImportTrackException { + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + var response = YtDlp.execute(ytDlpRequest); + // TODO: write to RedisProgressService + + TrackResponse result = null; + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + + if (downloadedFiles.isEmpty()) + throw new ImportTrackException("yt-dlp didn't downloaded anything for " + youtubeUrl); + + for (Path downloadedFile : downloadedFiles) { + var params = AddLocalTrackParamsBuilder.builder() + .blob(new PathBlob(downloadedFile)) + .ytdlpMetadata(OBJECT_MAPPER.writeValueAsString(videoInfo)) + .includeProgressHistory(false) + .build(); + + return params; + } + } + throw new ImportTrackException("cannot download any youtube track"); + } + + public List refreshYoutubePlaylist(CustomUserDetails user, long playlistId, long sourceId) + throws ImportTrackException { + Optional trackSourceMetadataOpt = trackSourceService.findWithMetadata(sourceId); + if (trackSourceMetadataOpt.isEmpty()) + throw new ImportTrackException("cannot find track source with metadata with id " + sourceId); + TrackSourceMetadata trackSourceMetadata = trackSourceMetadataOpt.get(); + String youtubeUrl = trackSourceMetadata.getUrl(); + + List videoInfos = Collections.emptyList(); + try { + videoInfos = YtDlp.getVideoInfo(youtubeUrl); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot `yt-dlp --dump-json` from " + youtubeUrl, e); + } + return refreshYoutubePlaylist(user.getId(), playlistId, trackSourceMetadata.getSource(), videoInfos, + youtubeUrl); + } + + public List refreshYoutubePlaylist(long userId, long playlistId, TrackSource trackSource, + List videoInfos, + String youtubeUrl) throws ImportTrackException { + List result = new ArrayList<>(); + logger.info(trackSource.getSourceUrl()); + try { + Path temporaryFolder = Files.createTempDirectory("yt-dlp-tmp"); + logger.info("temporaryFolder created {}", temporaryFolder.toString()); + + String downloadedMetadataKey = trackSource.getSourceUrl() + DOWNLOADED_METADATA_FILE; + try { + var rawBody = s3StorageService + .readRaw(downloadedMetadataKey); + Files.write(temporaryFolder.resolve(DOWNLOADED_METADATA_FILE), rawBody); + } catch (NoSuchKeyException e) { + logger.warn(".downloaded metadata file was not found, ignoring"); + } + + var ytDlpRequest = new YtDlpRequest(youtubeUrl, temporaryFolder.toAbsolutePath().toString()); + ytDlpRequest.setOption("output", "%(id)s"); + ytDlpRequest.setOption("download-archive", DOWNLOADED_METADATA_FILE); + ytDlpRequest.setOption("extract-audio"); + ytDlpRequest.setOption("audio-quality", 0); + ytDlpRequest.setOption("audio-format", "best"); + ytDlpRequest.setOption("no-overwrites"); + + PlaylistProgress playlistProgress = new PlaylistProgress(playlistId, trackSource.getId(), userId, + videoInfos.size()); + redisProgressService.saveProgress(playlistProgress); + + var response = YtDlp.execute(ytDlpRequest, (downloadProgress, ignored) -> { + redisProgressService.updateTrackProgressField(playlistId, trackSource.getId(), userId, + progress -> { + progress.setOverallProgress((int) downloadProgress); + }); + + }, stdoutLine -> { + redisProgressService.updateTrackProgressField(playlistId, trackSource.getId(), userId, + progress -> { + progress.setYtdlnStdout(String.join("\n", progress.getYtdlnStdout(), stdoutLine)); + }); + }, null); + redisProgressService.updateTrackProgressField(playlistId, trackSource.getId(), userId, + progress -> { + progress.setOverallProgress(100); + progress.setStatus(ProgressStatus.FINISHED); + }); + logger.info("yt dlp response {}", response); + + try (Stream pathStream = Files.walk(temporaryFolder)) { + List downloadedFiles = Files.walk(temporaryFolder).toList(); + logger.info("downloaded file count {}", downloadedFiles.size()); + + for (Path path : downloadedFiles) { + if (Files.isDirectory(path)) + continue; + boolean isMetadataFile = path.getFileName().toString().equals(DOWNLOADED_METADATA_FILE); + var body = Files.readAllBytes(path); + + if (isMetadataFile) { + s3StorageService.store(downloadedMetadataKey, body, Map.of()); + continue; + } + String fileName = fileNameWithoutExtension(path.getFileName().toString()); + VideoInfo videoInfo = videoInfos.stream() + .filter(v -> v.getId().equals(fileName)) + .findFirst() + .orElseThrow(); + + String audioKey = trackSource.getSourceUrl() + UUID.randomUUID().toString(); + + logger.info("downloaded file {} and info {}, key {}", fileName, videoInfo.getTitle(), audioKey); + + Optional metadata = Optional.empty(); + + try (var inputStream = Files.newInputStream(path)) { + metadata = metadataParseService.extractMetadata(inputStream); + } catch (IOException e) { + throw new ImportTrackException("cannot extract metadata from " + path.toString()); + } + String ffprobeJson = metadata.map(Metadata::rawJson).orElse("{}"); + + TrackSource playlistEntrySource; + try { + playlistEntrySource = trackSourceService.createTrackSourceWithKey(audioKey, body, ffprobeJson, + OBJECT_MAPPER.writeValueAsString(videoInfo), SourceType.PLAYLIST_ITEM); + } catch (IOException e) { + throw new ImportTrackException("cannot read blob body", e); + } + + Track track = trackRepository.save(new Track(playlistEntrySource)); + + String title = videoInfo.getTitle(); + String artist = metadata.map(Metadata::artist).orElse(null); + int durationSeconds = metadata.map(Metadata::durationSeconds).map(Float::intValue).orElse(0); + // TODO: thumbnail + // TODO: Recognize music if the duration is less than five minutes + // (configurable), and if not, it is a playlist and should be marked as is + trackMetadataService.createTrackMetadata( + track, title, fileName, audioKey, artist, null, durationSeconds); + + trackPlaylistService.insertTrackAtEnd(playlistId, track.getId()); + + String fileFormat = "unknown"; + if (metadata.isPresent()) { + fileFormat = metadata.map(m -> m.formatName()).get(); + } + + var trackResponse = new TrackResponse( + track.getId(), + title, + artist, + audioKey, + fileFormat, + durationSeconds, + fileName); + result.add(trackResponse); + } + } + return result; + } catch (IOException e) { + throw new ImportTrackException("i/o during playlist youtube video downloading", e); + } catch (YtDlpException e) { + throw new ImportTrackException("cannot download youtube video " + youtubeUrl, e); + } + } + + private String fileNameWithoutExtension(String fileName) { + return fileName.replaceFirst("[.][^.]+$", ""); + } + +} diff --git a/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java b/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java index bfaeab8..e96d64e 100644 --- a/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java +++ b/src/main/java/com/bivashy/backend/composer/service/importing/RedisMessageSubscriber.java @@ -1,28 +1,35 @@ package com.bivashy.backend.composer.service.importing; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.redis.connection.Message; import org.springframework.data.redis.connection.MessageListener; import org.springframework.data.redis.listener.ChannelTopic; import org.springframework.data.redis.listener.RedisMessageListenerContainer; import org.springframework.stereotype.Component; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; import com.bivashy.backend.composer.dto.importing.ImportTrackKey; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Consumer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; @Component public class RedisMessageSubscriber { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Logger logger = LoggerFactory.getLogger(Logger.class); private final RedisMessageListenerContainer container; - private final Map> subscriptions = new ConcurrentHashMap<>(); + private final Map> subscriptions = new ConcurrentHashMap<>(); public RedisMessageSubscriber(RedisMessageListenerContainer container) { this.container = container; } - public void subscribeToPlaylist(long playlistId, long userId, Consumer messageHandler) { + public void subscribeToPlaylist(long playlistId, long userId, Consumer messageHandler) { String channel = ImportTrackKey.redisChannelKey(playlistId, userId); String subscriptionKey = ImportTrackKey.subscriptionKey(playlistId, userId); @@ -32,7 +39,13 @@ public class RedisMessageSubscriber { public void onMessage(Message message, byte[] pattern) { String receivedMessage = new String(message.getBody()); if (subscriptions.containsKey(subscriptionKey)) { - messageHandler.accept(receivedMessage); + try { + BaseTrackProgress progress = OBJECT_MAPPER.readValue(receivedMessage, + BaseTrackProgress.class); + messageHandler.accept(progress); + } catch (JsonProcessingException e) { + logger.error("cannot deserialize message into BaseTrackProgress.class", e); + } } } }, new ChannelTopic(channel)); diff --git a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java index d3e70ab..8bc9851 100644 --- a/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java +++ b/src/main/java/com/bivashy/backend/composer/service/importing/RedisProgressService.java @@ -1,14 +1,22 @@ package com.bivashy.backend.composer.service.importing; -import com.bivashy.backend.composer.dto.importing.ImportTrackKey; -import com.bivashy.backend.composer.dto.importing.TrackProgressDTO; -import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import com.bivashy.backend.composer.dto.importing.BaseTrackProgress; +import com.bivashy.backend.composer.dto.importing.ImportTrackKey; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; @Service public class RedisProgressService { @@ -22,16 +30,16 @@ public class RedisProgressService { this.objectMapper = objectMapper; } - public void saveProgress(TrackProgressDTO progress) { + public void saveProgress(BaseTrackProgress progress) { try { String key = ImportTrackKey.progressKey(progress.getPlaylistId(), progress.getUserId()); String trackKey = ImportTrackKey.trackKey( progress.getPlaylistId(), - progress.getTrackId(), + progress.getTrackSourceId(), progress.getUserId()); String progressJson = objectMapper.writeValueAsString(progress); - redisTemplate.opsForHash().put(key, Long.toString(progress.getTrackId()), progressJson); + redisTemplate.opsForHash().put(key, Long.toString(progress.getTrackSourceId()), progressJson); redisTemplate.opsForValue().set(trackKey, progressJson); @@ -44,16 +52,48 @@ public class RedisProgressService { } } - public List getPlaylistProgress(long playlistId, long userId) { + public void updateTrackProgressField(long playlistId, long trackSourceId, long userId, + Consumer updater) { + try { + String trackKey = ImportTrackKey.trackKey(playlistId, trackSourceId, userId); + String hashKey = ImportTrackKey.progressKey(playlistId, userId); + + String existingJson = redisTemplate.opsForValue().get(trackKey); + if (existingJson == null) { + throw new RuntimeException("Track progress not found"); + } + + JavaType progressType = objectMapper.getTypeFactory() + .constructType(BaseTrackProgress.class); + + T progress = objectMapper.readValue(existingJson, progressType); + + updater.accept(progress); + + String updatedJson = objectMapper.writeValueAsString(progress); + redisTemplate.opsForHash().put(hashKey, Long.toString(trackSourceId), updatedJson); + redisTemplate.opsForValue().set(trackKey, updatedJson); + + redisTemplate.expire(hashKey, 24, TimeUnit.HOURS); + redisTemplate.expire(trackKey, 24, TimeUnit.HOURS); + + publishProgressUpdate(progress); + + } catch (Exception e) { + throw new RuntimeException("Failed to update track progress", e); + } + } + + public List getPlaylistProgress(long playlistId, long userId) { try { String key = ImportTrackKey.progressKey(playlistId, userId); Map progressMap = redisTemplate.opsForHash().entries(key); - List progressList = new ArrayList<>(); + List progressList = new ArrayList<>(); for (Object value : progressMap.values()) { - TrackProgressDTO progress = objectMapper.readValue( + BaseTrackProgress progress = objectMapper.readValue( (String) value, - TrackProgressDTO.class); + BaseTrackProgress.class); progressList.add(progress); } @@ -65,13 +105,13 @@ public class RedisProgressService { } } - public TrackProgressDTO getTrackProgress(long playlistId, long trackId, long userId) { + public BaseTrackProgress getTrackProgress(long playlistId, long trackSourceId, long userId) { try { - String key = ImportTrackKey.trackKey(playlistId, trackId, userId); + String key = ImportTrackKey.trackKey(playlistId, trackSourceId, userId); String progressJson = redisTemplate.opsForValue().get(key); if (progressJson != null) { - return objectMapper.readValue(progressJson, TrackProgressDTO.class); + return objectMapper.readValue(progressJson, BaseTrackProgress.class); } return null; } catch (Exception e) { @@ -79,7 +119,7 @@ public class RedisProgressService { } } - private void publishProgressUpdate(TrackProgressDTO progress) { + private void publishProgressUpdate(BaseTrackProgress progress) { try { String channel = ImportTrackKey.redisChannelKey(progress.getPlaylistId(), progress.getUserId()); String message = objectMapper.writeValueAsString(progress); diff --git a/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java b/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java new file mode 100644 index 0000000..4757cb0 --- /dev/null +++ b/src/main/java/com/bivashy/backend/composer/util/SimpleBlob.java @@ -0,0 +1,64 @@ +package com.bivashy.backend.composer.util; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.springframework.web.multipart.MultipartFile; + +public interface SimpleBlob { + InputStream inputStream() throws IOException; + + byte[] body() throws IOException; + + String fileName(); + + public static class MultipartBlob implements SimpleBlob { + private final MultipartFile multipartFile; + + public MultipartBlob(MultipartFile multipartFile) { + this.multipartFile = multipartFile; + } + + @Override + public InputStream inputStream() throws IOException { + return multipartFile.getInputStream(); + } + + @Override + public byte[] body() throws IOException { + return multipartFile.getBytes(); + } + + @Override + public String fileName() { + return multipartFile.getOriginalFilename(); + } + + } + + public static class PathBlob implements SimpleBlob { + private final Path path; + + public PathBlob(Path path) { + this.path = path; + } + + @Override + public InputStream inputStream() throws IOException { + return Files.newInputStream(path); + } + + @Override + public byte[] body() throws IOException { + return Files.readAllBytes(path); + } + + @Override + public String fileName() { + return path.getFileName().toString(); + } + + } +} diff --git a/src/main/resources/db/migration/V1_10__create_base_tables.sql b/src/main/resources/db/migration/V1_10__create_base_tables.sql index 5c4c53b..262d2b7 100644 --- a/src/main/resources/db/migration/V1_10__create_base_tables.sql +++ b/src/main/resources/db/migration/V1_10__create_base_tables.sql @@ -1,80 +1,69 @@ -CREATE TABLE IF NOT EXISTS "users" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "name" varchar(500) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW() +CREATE TABLE IF NOT EXISTS users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE TABLE IF NOT EXISTS "source_type" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "name" varchar(500) NOT NULL +CREATE TYPE source_type_enum AS ENUM ( + 'VIDEO', 'PLAYLIST', 'PLAYLIST_ITEM', 'FILE', 'URL' ); -CREATE TABLE IF NOT EXISTS "track_source" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "source_url" varchar(500) NOT NULL, - "source_type_id" bigint NOT NULL, - "last_fetched_at" timestamp NOT NULL DEFAULT NOW(), - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_track_source_source_type_id" - FOREIGN KEY ("source_type_id") REFERENCES "source_type" ("id") +CREATE TABLE IF NOT EXISTS track_source ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_url TEXT NOT NULL, + source_type SOURCE_TYPE_ENUM NOT NULL, + last_fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE TABLE IF NOT EXISTS "track" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "source_id" bigint NOT NULL, - CONSTRAINT "fk_track_source_id" - FOREIGN KEY ("source_id") REFERENCES "track_source" ("id") +CREATE TABLE IF NOT EXISTS track_source_metadata ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_id BIGINT NOT NULL UNIQUE REFERENCES track_source ( + id + ) ON DELETE CASCADE, + url TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS "track_metadata" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "track_id" bigint NOT NULL, - "title" varchar(500) NOT NULL, - "file_name" varchar(500) NOT NULL, - "audio_path" varchar(500) NOT NULL, - "artist" varchar(500), - "thumbnail_path" varchar(500), - "duration_seconds" integer, - CONSTRAINT "fk_track_metadata_track_id" - FOREIGN KEY ("track_id") REFERENCES "track" ("id") +CREATE TABLE IF NOT EXISTS track ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES track_source (id) ON DELETE RESTRICT ); -CREATE TABLE IF NOT EXISTS "playlist" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "owner_id" bigint NOT NULL, - "title" varchar(500) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - "updated_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_playlist_owner_id" - FOREIGN KEY ("owner_id") REFERENCES "users" ("id"), - CONSTRAINT "uq_playlist_owner_title" - UNIQUE ("owner_id", "title") +CREATE TABLE IF NOT EXISTS track_metadata ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + title TEXT NOT NULL, + file_name TEXT NOT NULL, + audio_path TEXT NOT NULL, + artist TEXT, + thumbnail_path TEXT, + duration_seconds INTEGER ); -CREATE TABLE IF NOT EXISTS "playlist_track" ( - "playlist_id" bigint NOT NULL, - "track_id" bigint NOT NULL, - "order_index" numeric NOT NULL, - CONSTRAINT "pk_playlist_track_new" PRIMARY KEY ("playlist_id", "track_id"), - CONSTRAINT "fk_playlist_track_playlist_id_new" - FOREIGN KEY ("playlist_id") REFERENCES "playlist" ("id"), - CONSTRAINT "fk_playlist_track_track_id_new" - FOREIGN KEY ("track_id") REFERENCES "track" ("id") +CREATE TABLE IF NOT EXISTS playlist ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + owner_id BIGINT NOT NULL REFERENCES users (id) ON DELETE CASCADE, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (owner_id, title) ); -CREATE TABLE IF NOT EXISTS "track_version" ( - "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - "track_id" bigint NOT NULL, - "metadata_id" bigint NOT NULL, - "source_id" bigint NOT NULL, - "created_at" timestamp NOT NULL DEFAULT NOW(), - CONSTRAINT "fk_track_version_track_id" - FOREIGN KEY ("track_id") REFERENCES "track" ("id"), - CONSTRAINT "fk_track_version_metadata_id" - FOREIGN KEY ("metadata_id") REFERENCES "track_metadata" ("id"), - CONSTRAINT "fk_track_version_source_id" - FOREIGN KEY ("source_id") REFERENCES "track_source" ("id") +CREATE TABLE IF NOT EXISTS playlist_track ( + playlist_id BIGINT NOT NULL REFERENCES playlist (id) ON DELETE CASCADE, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + order_index DECIMAL NOT NULL, + PRIMARY KEY (playlist_id, track_id) ); +CREATE TABLE IF NOT EXISTS track_version ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + track_id BIGINT NOT NULL REFERENCES track (id) ON DELETE CASCADE, + metadata_id BIGINT NOT NULL REFERENCES track_metadata ( + id + ) ON DELETE CASCADE, + source_id BIGINT NOT NULL REFERENCES track_source (id) ON DELETE RESTRICT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/src/main/resources/db/migration/V1_20__insert_enums.sql b/src/main/resources/db/migration/V1_20__insert_enums.sql deleted file mode 100644 index 652794c..0000000 --- a/src/main/resources/db/migration/V1_20__insert_enums.sql +++ /dev/null @@ -1,9 +0,0 @@ -INSERT INTO "source_type" ("id", "name") -OVERRIDING SYSTEM VALUE -VALUES - (1, 'VIDEO'), - (2, 'PLAYLIST'), - (3, 'FILE'), - (4, 'URL') -ON CONFLICT ("id") DO NOTHING; -