From 7cdb3d4164b22273d7a38a3c7b59e5fead3bc134 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Wed, 16 Nov 2022 13:42:29 +0300 Subject: [PATCH] 2022.11.16.0 --- Home.md | 18 ++++++++++++++++++ Plugins.md | 16 ++++++++++------ Settings.md | 46 +++++++++++++++++++++++++++++++++++++++++++--- Users.md | 9 +++++++-- 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/Home.md b/Home.md index a5440df..c348505 100644 --- a/Home.md +++ b/Home.md @@ -264,6 +264,8 @@ To add a user to the collection, click ```Add to collection``` in the context me When you add a user to a collection, the user's data will be moved to the collection folder with the name you specified. +You can select multiple users and a collection to add users to this collection. + ### Merging collection files When you add a user to the collection, a new folder will be created in the collection folder named by pattern ```Site```_```UserName```. That is, each profile in the collection is stored separately. When you merge, all files in the profiles' folders are moved to the root of the collection. @@ -287,6 +289,22 @@ If you want to merge the collection. Files from the "Reddit_SomeUserName1" and "Twitter_SomeUserName1" folders will be moved to the "First collection" folder. +### Virtual mode + +You can create a virtual collection. This means that this collection will only exist in SCrawler. The collection folder will not be created. The user files of users included in this type of collection, will not be moved. + +You can also virtually add a user to a real collection. In this case, the user's files will not be moved to the collection folder. + +You can only add a user to a virtual collection in virtual mode. + +# Import users + +```Settings-Import users``` + +You can import users that are not in SCrawler from the specific folder. You can also scan existing SCrawler directories to find missing users. + +**This feature is not for importing users from the site. It's more like searching for missing users.** + # Black list The program has the ```Black list```, which included users you have added to. It is only a blacklist of the program and no impact on the site! diff --git a/Plugins.md b/Plugins.md index 03db239..59b7cb7 100644 --- a/Plugins.md +++ b/Plugins.md @@ -125,7 +125,7 @@ interface ISiteSettings Icon Icon {get;}; Image Image {get;}; string Site {get;}; - string GetUserUrl(string UserName,bool Channel); + string GetUserUrl(IPluginContentProvider User, bool Channel); ExchangeOptions IsMyUser(string UserURL); ExchangeOptions IsMyImageVideo(string URL); IEnumerable GetSpecialData(string URL); @@ -146,7 +146,7 @@ interface ISiteSettings void Reset(); void OpenSettingsForm(); void UserOptions(ref object Options, bool OpenForm); - string GetUserPostUrl(string UserID, string PostID); + string GetUserPostUrl(IPluginContentProvider User, IUserMedia Media); } ``` @@ -164,9 +164,9 @@ interface IPluginContentProvider : IDisposable string ID {get; set;}; bool ParseUserMediaOnly {get; set;}; string UserDescription {get; set;}; - List ExistingContentList {get; set;}; + List ExistingContentList {get; set;}; List TempPostsList {get; set;}; - List TempMediaList {get; set;}; + List TempMediaList {get; set;}; bool UserExists {get; set;}; bool UserSuspended {get; set;}; bool IsSavedPosts {get; set;}; @@ -260,9 +260,9 @@ You must return an initialized instance of the ```IPluginContentProvider``` obje ### GetUserPostUrl -```Function GetUserPostUrl(ByVal UserID As String, ByVal PostID As String) As String``` +```Function GetUserPostUrl(ByVal UserID As IPluginContentProvider, ByVal Media As IUserMedia) As String``` -Return post URL based on user ID and post ID. +Return post URL based on user and media. ### Load @@ -590,6 +590,10 @@ If your plugin needs to run on a separate thread add the ```SeparatedTasks``` cl If you want users to be able to control the number of download tasks in a thread, use this attribute. Only works with [```SeparatedTasks```](#separatedtasks) +## TaskGroup + +Group this plugin into a specific download group. Look at the code to see how to use this attribute. + ## SavedPosts If your plugin provides downloading of saved posts, add the ```SavedPosts``` class attribute. diff --git a/Settings.md b/Settings.md index 86a03ef..c677519 100644 --- a/Settings.md +++ b/Settings.md @@ -88,6 +88,8 @@ - [Instagram](#instagram-requirements) - [RedGifs](#reddit-requirements) - [TikTok](#tiktok-requirements) +- [PornHub](#pornhub-requirements) +- [XHamster](#xhamster-requirements) - [XVIDEOS](#xvideos-requirements) - [LPSG](#lpsg-requirements) @@ -199,6 +201,7 @@ The Reddit parser can parse data without cookies, but you can add it if you like - [```ix-ig-www-claim```](#how-to-find-instagram-authorization-headers) - ```Saved posts user``` +*Both Hash and Hash2 are user hashes. I have never seen them change.* ### How to find Instagram Hash 1. Open browser developer tools @@ -211,6 +214,12 @@ The Reddit parser can parse data without cookies, but you can add it if you like - https://www.instagram.com/graphql/query/?query_hash=54321edcba54321edcba54321edcba54&variables={"user_id":"123456789","include_chaining":true,"include_reel":false,"include_suggested_users":false,"include_logged_out_extras":false,"include_highlight_reels":true,"include_live_status":true} 1. Copy this and paste it into the corresponding (```Hash```, ```Hash 2```) field on the Instagram settings form. +**If you don't see the required hash:** + +**Refresh the page several times or go to the main page and return to the profile until you see the hash you need.** + +**NEVER use wrong credentials! I specially wrote there that there are two different credentials. Using the wrong credentials may result in your account being banned!** + ### How to find Instagram authorization headers 1. Open browser developer tools 1. Go to ```Network``` @@ -264,15 +273,46 @@ When the number of tagged posts exceeds the tagged posts limit, you will be aske ### TikTok limits -Unfortunately, I didn't understand how to use the 'Cursor' option to get the JSON response. In this case, I can only parse an HTML page that only provides the latest 30 videos. **If you know how to use the cursor option to get a JSON response, you might create an issue with your suggestion.** +Unfortunately, I didn't understand how to use the 'Cursor' option to get the JSON response. In this case, I can only parse an HTML page that only provides the latest 30 videos. **If you know how to use the cursor option to get a JSON response, you can post with your suggestion [here](https://github.com/AAndyProgram/SCrawler/issues/75).** + +## PornHub +- ```Download GIF``` - default for new users. +- ```Download GIFs as mp4``` - download gifs in 'mp4' format instead of native 'webm'. +- ```Photo ModelHub only``` - download photo only from ModelHub. Prornstar photos hosted on PornHub itself will not be downloaded. **Attention! Downloading photos hosted on PornHub is a very heavy job.** +- ```Saved posts user``` - personal profile username (to download saved posts) + +![PornHub settings](https://github.com/AAndyProgram/SCrawler/blob/main/ProgramScreenshots/SettingsSitePornHub.png) + +### PornHub requirements +- [ffmpeg](#ffmpeg) is required anyway (only for x64 program) +- [```Cookies```](#how-to-set-up-cookies) for downloading private videos and saved posts +- ```Saved posts user``` for downloading saved posts + +### PornHub additional information + +About videos. PornHub has at last three page views (as I've seen). If you find that videos from the account you want to download aren't downloading, you can create a new issue with that account address. + +About photos. Photo download problems have the lowest priority. You can still create an issue including the account address where photos are not downloading, but I can't tell you when I will have time to fix it. + +## XHamster +- ```Download UHD``` - Download UHD (4K) content + +![XHamster settings](https://github.com/AAndyProgram/SCrawler/blob/main/ProgramScreenshots/SettingsSiteXHamster.png) + +### XHamster requirements +- [ffmpeg](#ffmpeg) is required anyway (only for x64 program) +- [```Cookies```](#how-to-set-up-cookies) for downloading saved posts ## XVIDEOS -- ```Download UHD``` - Download UHD (4K) content +- ```Download UHD``` - Download UHD (4K) content. +- ```Playlist of saved videos``` - Your personal videos playlist to download as 'saved posts'. This playlist must be private (Visibility = ```Only me```). It also required cookies. This playlist must be entered by pattern: ```https://www.xvideos.com/favorite/01234567/playlistname```. ![XVIDEOS settings](https://github.com/AAndyProgram/SCrawler/blob/main/ProgramScreenshots/SettingsSiteXvideos.png) ### XVIDEOS requirements -- [ffmpeg](#ffmpeg) for downloading videos hosted on reddit (only for x64 program) +- [ffmpeg](#ffmpeg) is required anyway (only for x64 program) +- [```Cookies```](#how-to-set-up-cookies) for downloading private videos and saved posts +- ```Playlist of saved videos``` for downloading saved posts ## LPSG diff --git a/Users.md b/Users.md index d6ed722..6ea216b 100644 --- a/Users.md +++ b/Users.md @@ -8,9 +8,8 @@ In the main window, click the ```Add``` button to open the user creation form. - https://twitter.com/SomeUserName - Reddit: - https://reddit.com/user/SomeUserName + - https://reddit.com/u/SomeUserName - https://reddit.com/r/SomeSubredditName - - u/SomeUserName - - r/SomeSubredditName - Instagram - https://www.instagram.com/SomeUserName - RedGifs @@ -18,6 +17,10 @@ In the main window, click the ```Add``` button to open the user creation form. - XVIDEOS - https://www.xvideos.com/Account-Type/SomeUserName - Account-Type/SomeUserName + - PornHub + - https://www.pornhub.com/users/UserName + - XHamster + - https://xhamster.com/users/UserName - ```Friendly name``` - if configured, this name will be displayed in the user list instead of the username. - ```Special path``` - specify a path other than the default path for user data. - ```Sites combo box``` - site pointers. @@ -37,6 +40,8 @@ In the main window, click the ```Add``` button to open the user creation form. - ```Script``` - If the checkbox is checked, script to be executed after the user download is complete. If the text is empty, the script from the global settings will be executed. [How to use the script](https://github.com/AAndyProgram/SCrawler/wiki/Settings#how-to-use-the-script). If neither ```Temporary``` nor ```Favorite``` is checked, the user will be displayed in the regular site group. +**In collection editing mode checkboxes have three states. When loaded, an indeterminate state means that profiles have different values for this option. When you save, checkboxes that are in indeterminate state will not be applied.** + # Labels In this form, you can create labels and select them for users.