diff --git a/pom.xml b/pom.xml index d063eb7..1921581 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,37 @@ 3.8.2 + + + org.jsoup + jsoup + 1.15.3 + + + + + org.apache.httpcomponents + httpclient + + + + com.xuxueli + xxl-job-core + 2.4.1 + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.junit.jupiter + junit-jupiter-api + test + + diff --git a/src/main/java/top/baogutang/music/client/QQMusicClient.java b/src/main/java/top/baogutang/music/client/QQMusicClient.java index 116c5aa..b8b44b3 100644 --- a/src/main/java/top/baogutang/music/client/QQMusicClient.java +++ b/src/main/java/top/baogutang/music/client/QQMusicClient.java @@ -307,6 +307,9 @@ public class QQMusicClient implements ChannelClient>>>>>>>>>> xxl-job config init start <<<<<<<<<<"); + XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor(); + xxlJobSpringExecutor.setAdminAddresses(adminAddresses); + xxlJobSpringExecutor.setAppname(appName); + xxlJobSpringExecutor.setAccessToken(accessKey); + log.info(">>>>>>>>>>> xxl-job config init success <<<<<<<<<<"); + return xxlJobSpringExecutor; + } + + +} diff --git a/src/main/java/top/baogutang/music/domain/res/DailySong.java b/src/main/java/top/baogutang/music/domain/res/DailySong.java new file mode 100644 index 0000000..36c8d52 --- /dev/null +++ b/src/main/java/top/baogutang/music/domain/res/DailySong.java @@ -0,0 +1,30 @@ +package top.baogutang.music.domain.res; + +import lombok.Data; + +import java.io.Serializable; + +/** + * + * @description: + * + * @author: N1KO + * @date: 2024/12/23 : 09:53 + */ +@Data +public class DailySong implements Serializable { + + private static final long serialVersionUID = 8341643795457014497L; + + private Long albumId; + + private String albumMid; + + private String albumName; + + private Long songId; + + private String songMid; + + private String songName; +} diff --git a/src/main/java/top/baogutang/music/properties/QQMusicProperties.java b/src/main/java/top/baogutang/music/properties/QQMusicProperties.java index 8aaf497..ca29b5f 100644 --- a/src/main/java/top/baogutang/music/properties/QQMusicProperties.java +++ b/src/main/java/top/baogutang/music/properties/QQMusicProperties.java @@ -26,6 +26,12 @@ public class QQMusicProperties { private String downloadBaseUrl2; + private String dailyMusicBaseUrl; + + private String dailyMusicSearchUrl; + + private String cookie; + private String downloadPath; } diff --git a/src/main/java/top/baogutang/music/schedule/DailyMusicHandler.java b/src/main/java/top/baogutang/music/schedule/DailyMusicHandler.java new file mode 100644 index 0000000..d371ed3 --- /dev/null +++ b/src/main/java/top/baogutang/music/schedule/DailyMusicHandler.java @@ -0,0 +1,46 @@ +package top.baogutang.music.schedule; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import top.baogutang.music.domain.res.DailySong; +import top.baogutang.music.enums.ChannelEnum; +import top.baogutang.music.service.IMusicService; +import top.baogutang.music.utils.JacksonUtil; + +import javax.annotation.Resource; +import java.util.List; + +/** + * + * @description: + * + * @author: N1KO + * @date: 2024/12/23 : 09:50 + */ +@Slf4j +public abstract class DailyMusicHandler { + + @Resource + private IMusicService musicService; + + public void fetchDailyAndSave() { + List songList = fetchDaily(); + if (CollectionUtils.isEmpty(songList)) { + return; + } + songList.forEach(song -> { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + log.error(">>>>>>>>>>process daily music error! channel:{},song:{}<<<<<<<<<<", getChannel(), JacksonUtil.toJson(song)); + return; + } + log.info(">>>>>>>>>>channel:{} start process daily music:{}<<<<<<<<<<", getChannel(), JacksonUtil.toJson(song)); + musicService.getMusicService(getChannel().getCode()).download(song.getSongMid()); + }); + } + + abstract List fetchDaily(); + + abstract ChannelEnum getChannel(); +} diff --git a/src/main/java/top/baogutang/music/schedule/QQDailyMusicHandler.java b/src/main/java/top/baogutang/music/schedule/QQDailyMusicHandler.java new file mode 100644 index 0000000..c4571d6 --- /dev/null +++ b/src/main/java/top/baogutang/music/schedule/QQDailyMusicHandler.java @@ -0,0 +1,199 @@ +package top.baogutang.music.schedule; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import top.baogutang.music.domain.res.DailySong; +import top.baogutang.music.enums.ChannelEnum; +import top.baogutang.music.exceptions.BusinessException; +import top.baogutang.music.properties.QQMusicProperties; +import top.baogutang.music.utils.JacksonUtil; +import top.baogutang.music.utils.OkHttpUtil; + +import javax.annotation.Resource; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; + +/** + * + * @description: + * + * @author: N1KO + * @date: 2024/12/23 : 09:51 + */ +@Slf4j +@Component +public class QQDailyMusicHandler extends DailyMusicHandler { + + @Resource + private QQMusicProperties qqMusicProperties; + + @Override + List fetchDaily() { + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.COOKIE, "ownCookie=" + qqMusicProperties.getCookie()); + headers.add(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"); + headers.add(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate"); + ResponseEntity response = OkHttpUtil.get(qqMusicProperties.getDailyMusicBaseUrl(), headers, byte[].class); + byte[] pageBytes = response.getBody(); + HttpHeaders responseHeaders = response.getHeaders(); + String contentEncoding = responseHeaders.getFirst(HttpHeaders.CONTENT_ENCODING); + String pageContent = null; + try { + pageContent = decompressResponse(pageBytes, contentEncoding); + } catch (IOException e) { + log.error(">>>>>>>>>>decompress response error:{}<<<<<<<<<<", e.getMessage(), e); + throw new BusinessException(e.getMessage()); + } + if (StringUtils.isBlank(pageContent)) { + log.error(">>>>>>>>>>query qq daily music content is empty<<<<<<<<<<"); + return Collections.emptyList(); + } + // 使用 Jsoup 解析 HTML + Document doc = Jsoup.parse(pageContent); + Element firstList = doc.selectFirst(".mod_for_u .playlist__item"); + String id = ""; + if (firstList != null) { + Element nameElement = firstList.selectFirst(".playlist__name"); + if (nameElement != null && "今日私享".equals(nameElement.text())) { + Element linkElement = firstList.selectFirst(".playlist__link"); + if (linkElement != null) { + id = linkElement.attr("data-rid"); + } + } + } + return parseFromId(id); + + } + + @Override + ChannelEnum getChannel() { + return ChannelEnum.QQ_MUSIC; + } + + private List parseFromId(String id) { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.add(HttpHeaders.REFERER, "https://y.qq.com/n/yqq/playlist"); + headers.add(HttpHeaders.COOKIE, "ownCookie=" + qqMusicProperties.getCookie()); + // 构建请求体 + ResponseEntity response = OkHttpUtil.get(String.format(qqMusicProperties.getDailyMusicSearchUrl(), id), headers, String.class); + String body = response.getBody(); + if (StringUtils.isBlank(body)) { + return null; + } + String bodyJson = extractJson(body, "jsonCallback"); + QQDailyCommonRes qqDailyCommonRes = JacksonUtil.fromJson(bodyJson, QQDailyCommonRes.class); + if (Objects.isNull(qqDailyCommonRes) || Objects.isNull(qqDailyCommonRes.getCdList()) || CollectionUtils.isEmpty(qqDailyCommonRes.getCdList())) { + return null; + } + return qqDailyCommonRes.getCdList().stream() + .map(cdList -> cdList.getSongList().stream() + .map(s -> { + DailySong dailySong = new DailySong(); + dailySong.setAlbumId(s.getAlbumId()); + dailySong.setAlbumMid(s.getAlbumMid()); + dailySong.setAlbumName(s.getAlbumName()); + dailySong.setSongId(s.getSongId()); + dailySong.setSongMid(s.getSongMid()); + dailySong.setSongName(s.getSongName()); + return dailySong; + }) + .collect(Collectors.toList())) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } + + private static String extractJson(String jsonp, String callback) { + String regex = "^" + Pattern.quote(callback) + "\\((.*)\\)$"; + Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); + Matcher matcher = pattern.matcher(jsonp); + if (matcher.find()) { + return matcher.group(1); + } + return null; + } + + private String decompressResponse(byte[] data, String contentEncoding) throws IOException { + if ("gzip".equalsIgnoreCase(contentEncoding)) { + try (GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(data))) { + return new String(gis.readAllBytes(), StandardCharsets.UTF_8); + } + } else if ("deflate".equalsIgnoreCase(contentEncoding)) { + try (InflaterInputStream iis = new InflaterInputStream(new ByteArrayInputStream(data))) { + return new String(iis.readAllBytes(), StandardCharsets.UTF_8); + } + } else { + // 未压缩或其他编码类型 + return new String(data, StandardCharsets.UTF_8); + } + } + + @Data + public static class QQDailyCommonRes implements Serializable { + + private static final long serialVersionUID = 1221533727687642619L; + + private Integer code; + + @JsonProperty("subcode") + private Integer subCode; + + @JsonProperty("cdlist") + private List cdList; + } + + @Data + public static class CdList implements Serializable { + + private static final long serialVersionUID = -6612510307776519521L; + + @JsonProperty("songids") + private String songIdList; + + @JsonProperty("songlist") + private List songList; + } + + @Data + public static class Song implements Serializable { + + private static final long serialVersionUID = -6515525575728411341L; + + @JsonProperty("albumid") + private Long albumId; + + @JsonProperty("albummid") + private String albumMid; + + @JsonProperty("albumname") + private String albumName; + + @JsonProperty("songid") + private Long songId; + + @JsonProperty("songmid") + private String songMid; + + @JsonProperty("songname") + private String songName; + } + +} diff --git a/src/main/java/top/baogutang/music/schedule/QQDailyMusicSchedule.java b/src/main/java/top/baogutang/music/schedule/QQDailyMusicSchedule.java new file mode 100644 index 0000000..ef32d44 --- /dev/null +++ b/src/main/java/top/baogutang/music/schedule/QQDailyMusicSchedule.java @@ -0,0 +1,27 @@ +package top.baogutang.music.schedule; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +/** + * + * @description: + * + * @author: N1KO + * @date: 2024/12/23 : 14:27 + */ +@Slf4j +@Component +public class QQDailyMusicSchedule { + + @Resource + private QQDailyMusicHandler dailyMusicHandler; + + @Scheduled(cron = "0 0 1 * * ?") + public void execute() { + dailyMusicHandler.fetchDailyAndSave(); + } +} diff --git a/src/main/java/top/baogutang/music/utils/OkHttpUtil.java b/src/main/java/top/baogutang/music/utils/OkHttpUtil.java index c72c044..7e2692d 100644 --- a/src/main/java/top/baogutang/music/utils/OkHttpUtil.java +++ b/src/main/java/top/baogutang/music/utils/OkHttpUtil.java @@ -4,6 +4,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; import okhttp3.*; import org.apache.commons.lang3.StringUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; import java.util.Map; import java.util.Objects; @@ -24,13 +32,21 @@ public class OkHttpUtil { return Singleton.INSTANCE.getInstance(); } + public static ResponseEntity post(String url, HttpEntity> entity, Class responseType) { + RestTemplate restTemplate = Singleton.INSTANCE.getRestTemplate(); + return restTemplate.postForEntity(url, entity, responseType); + } + private enum Singleton { /** * */ INSTANCE; + private final OkHttpClient singleton; + private final RestTemplate restTemplate; + Singleton() { OkHttpClient.Builder builder = new OkHttpClient.Builder(); builder.connectTimeout(10L, TimeUnit.SECONDS); @@ -39,11 +55,20 @@ public class OkHttpUtil { ConnectionPool connectionPool = new ConnectionPool(50, 60, TimeUnit.SECONDS); builder.connectionPool(connectionPool); singleton = builder.build(); + CloseableHttpClient httpClient = HttpClients.custom() + .useSystemProperties() + .build(); + HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient); + restTemplate = new RestTemplate(requestFactory); } public OkHttpClient getInstance() { return singleton; } + + public RestTemplate getRestTemplate() { + return restTemplate; + } } @@ -92,6 +117,12 @@ public class OkHttpUtil { return null; } + public static ResponseEntity get(String url, HttpHeaders headers, Class responseType) { + HttpEntity entity = new HttpEntity<>(headers); + RestTemplate restTemplate = Singleton.INSTANCE.getRestTemplate(); + return restTemplate.exchange(url, HttpMethod.GET, entity, responseType); + } + public static T get(String url, Map headerMap, Map params, TypeReference type) { try { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b0ce257..f99ee11 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -57,6 +57,9 @@ baogutang: album-base-url: http://117.72.78.133:5175/album/songs?albummid=%s download-base-url1: http://117.72.78.133:5176/song?url=https://y.qq.com/n/ryqq/songDetail/%s download-base-url2: http://117.72.78.133:5176/song?url=https://u.y.qq.com/n/ryqq/songDetail/%s + daily-music-base-url: https://c.y.qq.com/node/musicmac/v6/index.html + cookie: "RK=rUvZdxsXGU; ptcz=da14a0149395412d1ccada68745ff33321cfa01aad39b07aa3fcecae55395d97; pac_uid=0_eTHZsZHX7ba4M; _qimei_uuid42=18b140b1a2a100af5a8d7679cb48e01d678edc347b; _qimei_fingerprint=caa870019e5e1e0d9dc30786b07078ba; _qimei_h38=e25542625a8d7679cb48e01d03000006d18b14; _qimei_q36=; pgv_pvid=7363669324; fqm_pvqid=7c5bc23f-c99a-4341-ba32-643f6cfc30f2; ts_refer=www.google.com.hk/; ts_uid=3998702427; fqm_sessionid=ab065cb0-5652-4f77-a339-baf6a8afdb5f; pgv_info=ssid=s5380242360; login_type=1; ts_last=y.qq.com/n/ryqq/search; wxopenid=; uin=940879015; euin=NKvzNeSqoe6k; music_ignore_pskey=202306271436Hn@vBj; psrf_qqunionid=538FF9C402100468B3B9CB56FB49CA8B; wxunionid=; tmeLoginType=2; qqmusic_key=Q_H_L_63k3NJFbLtBHdyzgGBtfNP6_ZAiH0BPaf9vLQMUFOLe-WVnuOWGp-JUh7sgP-oSjrmFnzawQiTS69LcLVMFvs9w; psrf_qqopenid=EBFCD19499A42B5DFF06DE7BB15B3D98; psrf_qqrefresh_token=E2D138702A05B6A1D0582E516C18B595; qm_keyst=Q_H_L_63k3NJFbLtBHdyzgGBtfNP6_ZAiH0BPaf9vLQMUFOLe-WVnuOWGp-JUh7sgP-oSjrmFnzawQiTS69LcLVMFvs9w; psrf_musickey_createtime=1734917905; psrf_qqaccess_token=372F8B12E8910A482D80FDB5E3190D75; wxrefresh_token=; psrf_access_token_expiresAt=1735522705" + daily-music-search-url: http://c.y.qq.com/qzone/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&utf8=1&loginUin=0&disstid=%s download-path: /downloads/music # download-path: /Users/nikooh/Desktop/downloads/music @@ -70,4 +73,13 @@ baogutang: # download-path: /Users/nikooh/Desktop/downloads/music +xxl: + job: + admin: + addresses: http://117.72.78.133:8900/xxl-job-admin + executor: + appName: ${spring.application.name} + accessKey: WJtjy1217 + + diff --git a/src/test/java/top/baogutang/music/schedule/DailyMusicHandlerTest.java b/src/test/java/top/baogutang/music/schedule/DailyMusicHandlerTest.java new file mode 100644 index 0000000..a4dd84e --- /dev/null +++ b/src/test/java/top/baogutang/music/schedule/DailyMusicHandlerTest.java @@ -0,0 +1,30 @@ +package top.baogutang.music.schedule; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import top.baogutang.music.domain.res.DailySong; + +import javax.annotation.Resource; +import java.util.List; + +/** + * + * @description: + * + * @author: N1KO + * @date: 2024/12/23 : 10:15 + */ +@Slf4j +@SpringBootTest +class DailyMusicHandlerTest { + + @Resource + private QQDailyMusicHandler dailyMusicHandler; + + @Test + void test_daily_handler() { + dailyMusicHandler.fetchDailyAndSave(); + } +}