下载抖音用户主页数据!
当前为
// ==UserScript==
// @name douyin-user-data-download
// @namespace http://tampermonkey.net/
// @version 0.3.3
// @description 下载抖音用户主页数据!
// @author xxmdmst
// @match https://www.douyin.com/*
// @icon https://xxmdmst.oss-cn-beijing.aliyuncs.com/imgs/favicon.ico
// @grant none
// @require https://cdnjs.cloudflare.com/ajax/libs/jszip/3.6.0/jszip.min.js
// @license MIT
// ==/UserScript==
(function() {
let table;
function initGbkTable() {
// https://en.wikipedia.org/wiki/GBK_(character_encoding)#Encoding
const ranges = [
[0xA1, 0xA9, 0xA1, 0xFE],
[0xB0, 0xF7, 0xA1, 0xFE],
[0x81, 0xA0, 0x40, 0xFE],
[0xAA, 0xFE, 0x40, 0xA0],
[0xA8, 0xA9, 0x40, 0xA0],
[0xAA, 0xAF, 0xA1, 0xFE],
[0xF8, 0xFE, 0xA1, 0xFE],
[0xA1, 0xA7, 0x40, 0xA0],
];
const codes = new Uint16Array(23940);
let i = 0;
for (const [b1Begin, b1End, b2Begin, b2End] of ranges) {
for (let b2 = b2Begin; b2 <= b2End; b2++) {
if (b2 !== 0x7F) {
for (let b1 = b1Begin; b1 <= b1End; b1++) {
codes[i++] = b2 << 8 | b1
}
}
}
}
table = new Uint16Array(65536);
table.fill(0xFFFF);
const str = new TextDecoder('gbk').decode(codes);
for (let i = 0; i < str.length; i++) {
table[str.charCodeAt(i)] = codes[i]
}
}
function str2gbk(str, opt = {}) {
if (!table) {
initGbkTable()
}
const NodeJsBufAlloc = typeof Buffer === 'function' && Buffer.allocUnsafe;
const defaultOnAlloc = NodeJsBufAlloc
? (len) => NodeJsBufAlloc(len)
: (len) => new Uint8Array(len);
const defaultOnError = () => 63;
const onAlloc = opt.onAlloc || defaultOnAlloc;
const onError = opt.onError || defaultOnError;
const buf = onAlloc(str.length * 2);
let n = 0;
for (let i = 0; i < str.length; i++) {
const code = str.charCodeAt(i);
if (code < 0x80) {
buf[n++] = code;
continue
}
const gbk = table[code];
if (gbk !== 0xFFFF) {
buf[n++] = gbk;
buf[n++] = gbk >> 8
} else if (code === 8364) {
buf[n++] = 0x80
} else {
const ret = onError(i, str);
if (ret === -1) {
break
}
if (ret > 0xFF) {
buf[n++] = ret;
buf[n++] = ret >> 8
} else {
buf[n++] = ret
}
}
}
return buf.subarray(0, n)
}
let aweme_list = [];
let numMsg1,numMsg2;
let userKey = [
"昵称", "关注", "粉丝", "获赞",
"抖音号", "IP属地", "性别",
"位置", "签名", "作品数", "主页"
];
let userData = [];
let timer, dimg_button;
function createVideoButton(text, top, func) {
const button = document.createElement("button");
button.textContent = text;
button.style.position = "absolute";
button.style.right = "0px";
button.style.top = top;
button.style.opacity = "0.5";
button.addEventListener("click", func);
return button;
}
function openLink(url) {
const link = document.createElement('a');
link.href = url;
link.target = "_blank";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
function createDownloadButton() {
let targetNodes = document.querySelectorAll("div[data-e2e='user-post-list'] > ul[data-e2e='scroll-list'] > li a");
for (let i = 0; i < targetNodes.length; i++) {
let targetNode = targetNodes[i];
if (targetNode.dataset.added) {
continue;
}
// const button2 = createVideoButton("复制链接", "0px", (event) => {
// event.preventDefault();
// event.stopPropagation();
// navigator.clipboard.writeText(aweme_list[i].url).then(() => {
// button2.textContent = "复制成功";
// }).catch((e) => {
// button2.textContent = "复制失败";
// });
// setTimeout(() => button2.textContent = '复制链接', 2000);
// });
// targetNode.appendChild(button2);
const button3 = createVideoButton("打开视频源", "0px", (event) => {
event.preventDefault();
event.stopPropagation();
openLink(aweme_list[i].url);
});
targetNode.appendChild(button3);
const button = createVideoButton("下载视频", "21px", (event) => {
event.preventDefault();
event.stopPropagation();
let xhr = new XMLHttpRequest();
xhr.open('GET', aweme_list[i].url.replace("http://", "https://"), true);
xhr.responseType = 'blob';
xhr.onload = (e) => {
let a = document.createElement('a');
a.href = window.URL.createObjectURL(xhr.response);
a.download = (aweme_list[i].desc ? aweme_list[i].desc.slice(0,20).replace(/[\/:*?"<>|\s]/g, "") : aweme_list[i].awemeId) + (aweme_list[i].images ? ".mp3" : ".mp4");
a.click()
};
xhr.onprogress = (event) => {
if (event.lengthComputable) {
button.textContent = "下载" + (event.loaded * 100 / event.total).toFixed(1) + '%';
}
};
xhr.send();
});
targetNode.appendChild(button);
if (aweme_list[i].images) {
const button4 = createVideoButton("图片打包下载", "42px", (event) => {
event.preventDefault();
event.stopPropagation();
const zip = new JSZip();
console.log(aweme_list[i].images);
button4.textContent = "下载并打包中...";
const promises = aweme_list[i].images.map((link, index) => {
return fetch(link)
.then((response) => response.arrayBuffer())
.then((buffer) => {
zip.file(`image_${index + 1}.jpg`, buffer);
});
});
Promise.all(promises)
.then(() => {
return zip.generateAsync({type: "blob"});
})
.then((content) => {
const link = document.createElement("a");
link.href = URL.createObjectURL(content);
link.download = (aweme_list[i].desc ? aweme_list[i].desc.slice(0,20).replace(/[\/:*?"<>|\s]/g, "") : aweme_list[i].awemeId) + ".zip";
link.click();
button4.textContent = "图片打包完成";
});
});
targetNode.appendChild(button4);
}
targetNode.dataset.added = true;
}
}
function interceptResponse() {
const originalSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function () {
const self = this;
this.onreadystatechange = function () {
if (self.readyState === 4) {
if (self._url.indexOf("/aweme/v1/web/aweme/post") > -1) {
var json = JSON.parse(self.response);
let post_data = json.aweme_list.map(item => Object.assign(
{"awemeId": item.aweme_id, "desc": item.desc.replace(/[^\x00-\x7F\u4E00-\u9FFF\uFF00-\uFFEF]+/g, " ").trim()},
{
"diggCount": item.statistics.digg_count,
"commentCount": item.statistics.comment_count,
"collectCount": item.statistics.collect_count,
"shareCount": item.statistics.share_count
},
{
"date": new Date(item.create_time * 1000).toLocaleString(),
"url": item.video.play_addr.url_list[0]
},
{
"images": item.images ? item.images.map(row => row.url_list.pop()) : null
}
));
aweme_list.push(...post_data);
numMsg1.innerText = `已加载${aweme_list.length}条`;
numMsg2.innerText = `图集${aweme_list.filter(a=>a.images).length}条`;
if (timer !== undefined)
clearTimeout(timer);
timer = setTimeout(createDownloadButton, 500);
dimg_button.textContent = "图文批量打包下载";
} else if(self._url.indexOf("/aweme/v1/web/user/profile/other") > -1){
var userInfo = JSON.parse(self.response).user;
userData.push(
userInfo.nickname, userInfo.following_count, userInfo.mplatform_followers_count,
userInfo.total_favorited, '\t' + (userInfo.unique_id ? userInfo.unique_id : userInfo.short_id), userInfo.ip_location,userInfo.gender===2?"女":"男",
`${userInfo.city}·${userInfo.district}`, '"' + (userInfo.signature ?userInfo.signature:'') + '"', userInfo.aweme_count, "https://www.douyin.com/user/" + userInfo.sec_uid
);
}
}
};
originalSend.apply(this, arguments);
};
}
interceptResponse();
function txt2file(txt, filename) {
const blob = new Blob([txt], {type: 'text/plain'});
const url = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
link.download = filename.slice(0,20).replace(/[\/:*?"<>|\s]/g, "");
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
URL.revokeObjectURL(url);
}
function downloadData(encoding) {
let text = userKey.join(",") + "\n" + userData.join(",") + "\n\n";
text += "作品描述,点赞数,评论数,收藏数,分享数,发布时间,下载链接\n";
aweme_list.forEach(item => {
text += ['"' + item.desc + '"', item.diggCount, item.commentCount,
item.collectCount, item.shareCount, item.date, item.url].join(",") + "\n"
});
if (encoding === "gbk"){
text = str2gbk(text);
}
txt2file(text, userData[0] + ".csv");
}
function createButton(title, top) {
top = top === undefined ? "60px" : top;
const button = document.createElement('button');
button.textContent = title;
button.style.position = 'fixed';
button.style.right = '5px';
button.style.top = top;
button.style.zIndex = '90000';
button.style.opacity = "0.5";
document.body.appendChild(button);
return button
}
function createDownloadAllData(){
const label = document.createElement('label');
label.setAttribute('for', 'gbk');
label.innerText = 'gbk';
label.style.position = 'fixed';
label.style.right = '86px';
label.style.top = '81px';
label.style.color = 'white';
label.style.zIndex = '90000';
label.style.opacity = "0.8";
const checkbox = document.createElement('input');
checkbox.setAttribute('type', 'checkbox');
checkbox.setAttribute('id', 'gbk');
checkbox.style.position = 'fixed';
checkbox.style.right = '106px';
checkbox.style.top = '84px';
checkbox.style.zIndex = '90000';
document.body.appendChild(label);
document.body.appendChild(checkbox);
createButton("下载已加载数据", "81px").addEventListener('click', (e) => downloadData(checkbox.checked?"gbk":""));
}
function createScrollPageToBottom() {
let scrollInterval;
function scrollLoop() {
let endText=document.querySelector("div[data-e2e='user-post-list'] > ul[data-e2e='scroll-list'] + div div").innerText;
if (aweme_list.length < userData[9] || !endText) {
scrollTo(0, document.body.scrollHeight);
} else {
clearInterval(scrollInterval);
scrollInterval=null;
button.textContent = "已加载全部！";
}
}
let button = createButton('开启自动下拉', '60px');
button.addEventListener('click', ()=>{
if(!scrollInterval){
scrollInterval = setInterval(scrollLoop, 1200);
button.textContent = "停止自动下拉";
} else {
clearInterval(scrollInterval);
scrollInterval=null;
button.textContent = "开启自动下拉";
}
});
numMsg1 = document.createElement('span');
numMsg1.innerText = '已加载0条';
numMsg1.style.color = 'white';
numMsg1.style.position = 'fixed';
numMsg1.style.right = '98px';
numMsg1.style.top = '60px';
numMsg1.style.color = 'white';
numMsg1.style.zIndex = '90000';
numMsg1.style.opacity = "0.5";
document.body.appendChild(numMsg1);
numMsg2 = document.createElement('span');
numMsg2.innerText = '';
numMsg2.style.color = 'white';
numMsg2.style.position = 'fixed';
numMsg2.style.right = '98px';
numMsg2.style.top = '102px';
numMsg2.style.color = 'white';
numMsg2.style.zIndex = '90000';
numMsg2.style.opacity = "0.5";
document.body.appendChild(numMsg2);
}
async function downloadImg() {
const zip = new JSZip();
let flag = true;
for (let [index, aweme] of aweme_list.filter(a=>a.images).entries()) {
dimg_button.textContent = `${index + 1}.${aweme.desc.slice(0,20)}...`;
let folder = zip.folder((index + 1) + "." + (aweme.desc ? aweme.desc.replace(/[\/:*?"<>|\s]/g, "").slice(0,20).replace(/[.\d]+$/g, "") : aweme.awemeId));
await Promise.all(aweme.images.map((link, index) => {
return fetch(link)
.then((res) => res.arrayBuffer())
.then((buffer) => {
folder.file(`image_${index + 1}.jpg`, buffer);
});
}));
flag = false;
}
if (flag) {
alert("当前页面未发现图文链接");
return
}
dimg_button.textContent = "图片打包中...";
zip.generateAsync({type: "blob"})
.then((content) => {
const link = document.createElement("a");
link.href = URL.createObjectURL(content);
link.download = userData[0].slice(0,20).replace(/[\/:*?"<>|\s]/g, "") + ".zip";
link.click();
dimg_button.textContent = "图片打包完成";
});
}
function douyinVideoDownloader() {
function run(){
let downloadOption = [{name:'打开视频源',id:'toLink'}];
let videoElements = document.querySelectorAll('video');
if(videoElements.length == 0) return;
//把自动播放的video标签选择出来
let playVideoElements=[];
videoElements.forEach(function(element){
let autoplay = element.getAttribute('autoplay');
if(autoplay !== null){
playVideoElements.push(element);
}
})
let videoContainer=location.href.indexOf('modal_id') != -1
? playVideoElements[0]
: playVideoElements[playVideoElements.length-1];
if(!videoContainer) return;
//获取视频播放地址
let url = videoContainer && videoContainer.children.length>0 && videoContainer.children[0].src
? videoContainer.children[0].src
: videoContainer.src;
//获取视频ID，配合自定义id使用
let videoId;
let resp = url.match(/^(https:)?\/\/.+\.com\/([a-zA-Z0-9]+)\/[a-zA-Z0-9]+\/video/);
let res = url.match(/blob:https:\/\/www.douyin.com\/(.*)/);
if(resp && resp[2]){
videoId=resp[2];
}else if(res && res[1]){
videoId=res[1]
}else{
videoId = videoContainer.getAttribute('data-xgplayerid')
}
let playContainer = videoContainer.parentNode.parentNode.querySelector('.xg-right-grid');
if(!playContainer) return;
//在对主页就行视频浏览时会出现多个按钮，删除不需要的，只保留当前对应的
let videoDownloadDom = playContainer.querySelector('#scriptVideoDownload'+videoId);
if(videoDownloadDom){
let dom = playContainer.querySelectorAll('.xgplayer-playclarity-setting');
dom.forEach(function(d){
let btn = d.querySelector('.btn');
if(d.id != 'scriptVideoDownload'+videoId && btn.innerText=='下载'){
d.parentNode.removeChild(d);
}
});
return;
}
if(videoContainer && playContainer){
let playClarityDom = playContainer.querySelector('.xgplayer-playclarity-setting');
if(!playClarityDom) return;
let palyClarityBtn = playClarityDom.querySelector('.btn');
if(!palyClarityBtn) return;
let downloadDom = playClarityDom.cloneNode(true);
downloadDom.setAttribute('id','scriptVideoDownload'+videoId);
if(location.href.indexOf('search') == -1){
downloadDom.style='margin-top:-68px;padding-top:100px;padding-left:20px;padding-right:20px;';
}else{
downloadDom.style='margin-top:0px;padding-top:100px;';
}
let downloadText = downloadDom.querySelector('.btn');
downloadText.innerText='下载';
downloadText.style = 'font-size:14px;font-weight:600;';
downloadText.setAttribute('id','zhmDouyinDownload'+videoId);
let detail = playContainer.querySelector('xg-icon:nth-of-type(1)').children[0];
let linkUrl = detail.getAttribute('href') ? detail.getAttribute('href') : location.href;
if(linkUrl.indexOf('www.douyin.com')==-1){
linkUrl='//www.douyin.com'+linkUrl;
}
downloadText.setAttribute('data-url',linkUrl);
downloadText.removeAttribute('target');
downloadText.setAttribute('href','javascript:void(0);');
let virtualDom = downloadDom.querySelector('.virtual');
downloadDom.onmouseover=function(){
if(location.href.indexOf('search') == -1){
virtualDom.style='display:block !important';
}else{
virtualDom.style='display:block !important;margin-bottom:37px;';
}
}
downloadDom.onmouseout=function(){
virtualDom.style='display:none !important';
}
let downloadHtml = '';
downloadOption.forEach(function(item){
if(item.id=="toLink"){
downloadHtml += `<div style="text-align:center;" class="item ${item.id}" id="${item.id}${videoId}">${item.name}</div>`;
}
})
if(downloadDom.querySelector('.virtual')){
downloadDom.querySelector('.virtual').innerHTML = downloadHtml;
}
playClarityDom.after(downloadDom);
//直接打开
let toLinkDom = playContainer.querySelector('#toLink'+videoId);
if(toLinkDom){
toLinkDom.addEventListener('click',function(){
if(url.match(/^blob/)){
commonFunctionObject.webToast({"message":"加密视频地址，无法直接打开", "background":"#141414"});
}else{
window.open(url);
}
})
}
}
}
setInterval(run, 500);
};
window.onload = () => {
douyinVideoDownloader();
if(location.href.startsWith("https://www.douyin.com/user/")){
createDownloadAllData();
createScrollPageToBottom();
dimg_button = createButton("图文批量打包下载", "102px");
dimg_button.addEventListener('click', downloadImg);
}
};
})();