From f445e4f5c513a958fa7d6c96313d5431a69394ad Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Fri, 17 Nov 2023 02:33:57 +0300 Subject: [PATCH] 2023.11.17.0 --- Home.md | 19 +++++++++++++++++++ Plugins.md | 38 +++++++++++++++++++++++++++++++++++--- Settings.md | 30 +++++++++++++++++++++++++----- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/Home.md b/Home.md index b2910e3..15246e0 100644 --- a/Home.md +++ b/Home.md @@ -92,6 +92,8 @@ This form will show the recently downloaded data (images and videos). Each image Double-clicking on an image will open the original image in an external viewer. +**You can also add posts to your Favorite feed and/or special feed(s) that you can create.** + ![Feed window](https://raw.githubusercontent.com/AAndyProgram/SCrawler/main/ProgramScreenshots/FeedWindow.png) Buttons: @@ -104,6 +106,10 @@ Context menu: - `Open user` (with folder image) - open user folder - `Open user` (with globe) - open user site - `Open post` - open post in browser +- `Add to Favorite` - this post will be added to your favorite feed +- `Add to special feed...`- this post will be added to the feed(s) you select (a selection form will open) +- `Remove from Favorite` - this post will be removed from your favorite feed +- `Remove from special feed...` - this post will be removed from the feed(s) you select (a selection form will open) - `Find user` - find user in the main window - `Information` - show post information - `Delete picture/video` - delete picture/video @@ -111,8 +117,20 @@ Context menu: ![Feed context menu](https://raw.githubusercontent.com/AAndyProgram/SCrawler/main/ProgramScreenshots/FeedWindowItemContext.png) Load sessions: +- `Load current session` - `Load last session` - load last saved session data - `Select loading session` - load the data of the session you have chosen (the session selection form will open) +- `Load Favorite` - load your Favorite feed +- `Load special feed` - load one or more of your special feeds +- `Add checked to Favorite` - add checked posts to your Favorite feed +- `Remove checked from Favorite` - remove checked posts from your Favorite feed +- `Add checked to special feed...` - add checked posts to one or more of your feeds +- `Remove checked from special feed...` - remove checked posts from one or more of your feeds +- `Clear Favorite` - remove all posts from your Favorite feed (just clearing the feed; **files will not be deleted**) +- `Clear special feed...` - remove all posts from one or more of your feeds (just clearing the feeds; **files will not be deleted**) +- `Delete special feed...`- delete one or more of your feeds (**files will not be deleted**) +- `Delete daily feed (by list)` - delete one or more session feeds (a selection form will open; **files will not be deleted**) +- `Delete daily feed (by date)` - delete session feeds by the date (the date selection form will open; **files will not be deleted**) Change view: - `Downloads` - show downloaded regular users @@ -300,6 +318,7 @@ If you want to add a folder as a user, you need to use the following pattern: `D - `Subscription` - read about subscriptions [here](#subscriptions) - `Sites combo box` - site pointers. - `Options` - additional user options. +- `Account` - if you've configured more than one account for the selected site, this field will be active. You can select the account through which the user will be downloaded. - `Temporary` - mark user as `Temporary`. - `Favorite` - mark user as `Favorite`. - `Download images/videos` - for the created user, only the selected media types will be downloaded. diff --git a/Plugins.md b/Plugins.md index 7704cac..4657734 100644 --- a/Plugins.md +++ b/Plugins.md @@ -149,32 +149,38 @@ Exit-->[*] ## ISiteSettings ``` CSharp -interface ISiteSettings +interface ISiteSettings: IDisposable { Icon Icon {get;}; Image Image {get;}; string Site {get;}; + string AccountName {get; set;}; + bool Temporary {get; set;}; + ISiteSettings DefaultInstance {get; set;}; bool SubscriptionsAllowed {get;}; ILogProvider Logger {get; set;}; string GetUserUrl(IPluginContentProvider User); ExchangeOptions IsMyUser(string UserURL); ExchangeOptions IsMyImageVideo(string URL); IPluginContentProvider GetInstance(Download What); - void Load(IEnumerable> XMLValues); IEnumerable> XMLFields(); void BeginInit(); void EndInit(); + string AvailableText {get; set;}; bool Available(); bool ReadyToDownload(); void DownloadStarted(Download What); void BeforeStartDownload(object User, Download What); void AfterDownload(object User, Download What); void DownloadDone(Download What); + ISiteSettings Clone(bool Full); + void Delete(); void BeginEdit(); void EndEdit(); void BeginUpdate(); void EndUpdate(); void Update(); + void Update(ISiteSettings Source); void Reset(); void OpenSettingsForm(); void UserOptions(ref object Options, bool OpenForm); @@ -184,8 +190,20 @@ interface ISiteSettings `Site`, `Icon` and `Image` are properties that provide the site name and site icon (in `Icon` and `Image` formats). +`AccountName` property set before calling the `BeginInit` function. + +`Temporary` property means that this instance is temporary. Use when cloning an existing instance or create a new one (before saving data). + +The `DefaultInstance` property will set to default instance for an instances that are not default. In the default instance, this value is `Nothing` (`null` in c#). + +`AvailableText` property must store the notification text (if it exists) when the `Available` function is called with the argument `Silent = True`. + `SubscriptionsAllowed` indicates that the plugin allows users to be created in subscription mode. +The `Clone` function must return a clone of the current instance in order to create a new one (based on the current one). + +The `Delete` function is called when the user deletes the current profile. + `IsMyUser` and `IsMyImageVideo` are URLs validators. These functions should return a value (`ExchangeOptions` with `Exists` set to `true`) indicating whether the user or media belongs to your plugin. `GetInstance` must return an instance for downloading data. The `What` parameter specifies what type of instance is being requested. @@ -196,6 +214,8 @@ interface ISiteSettings `BeginUpdate`, `EndUpdate` and `Update` are function that are called by the site editor form (when the plugin settings have changed and need to be saved) and by the settings class. +The `Update(Source)` function is called when the user has created a new instance and added it to SCrawler. The `Source` argument stores the data that must be copied to the current instance. + `Available` function must return a value indicating that the site is available. If false is returned, users will not be downloaded. **Called when jobs the pool is created.** `ReadyToDownload` function must return the same value as the `Available` function, but it is **called before particular user actually downloads**. @@ -221,6 +241,7 @@ interface IDownloadableMedia : IUserMedia, IDisposable Icon SiteIcon {get;}; string Site {get;}; string SiteKey {get;}; + string AccountName {get; set;}; string ThumbnailUrl {get; set;}; string ThumbnailFile {get; set;}; string Title {get; set;}; @@ -276,6 +297,7 @@ interface IPluginContentProvider : IDisposable IThrower Thrower {get; set;}; ILogProvider LogProvider {get; set;}; ISiteSettings Settings {get; set;}; + string AccountName {get; set;}; string Name {get; set;}; string ID {get; set;}; string Options {get; set;}; @@ -366,7 +388,7 @@ Functions: - `Message` - additional message - `ShowMainMsg` - show main message - `ShowErrorMsg` - show error message - - `SendtoLog` - message and error message will be sent to the program log + - `SendToLog` - message and error message will be sent to the program log # Objects @@ -381,6 +403,8 @@ There are three initialization constructors: If your initial value is null, you **MUST** set the type. +Use `BeginInit` and `EndInit` to pause handlers. Use `Clone` to properly clone a property using the `ISiteSettings` `Clone` and `Update` functions. + **[Only these types are available](#propertyvalue-value-types)** ## IPropertyValue @@ -441,6 +465,14 @@ This attribute specifies that your property should be added to the SCrawler sett Again. Only works with `PropertyValue` [object](#propertyvalue). +## DoNotUse + +This attribute allows you to exclude a property from the setting environment. Suitable for overrides. + +- `Value` - a value that indicates whether this field should be used or not. `True` = don't use; `False` = Use. + +Only works with `PropertyValue` [object](#propertyvalue). + ## PropertyUpdater This attribute provides a mechanism for updating a specific property. Attributed can only be applied to a method. Allows multiple definitions. Function **MUST** return a `Boolean` (`bool` in C#) with `true` indicating that the value has been updated and `false` otherwise. diff --git a/Settings.md b/Settings.md index b904ba0..d9d086c 100644 --- a/Settings.md +++ b/Settings.md @@ -102,6 +102,7 @@ - `Add missing information to log` - Text will be added to the log stating that the [missing posts](https://github.com/AAndyProgram/SCrawler/wiki/#missing-posts) exist. - `Add missing errors to log` - Each error that prevents SCrawler from downloading a file will be added to the log. - `Trying to download missing posts using regular download` - If missing posts exist, the missing posts will attempt to be downloaded via user download. +- `Use the default account if the selected account does not exist` - Use the default account if you deleted an account that you used for some users. ![Downloading settings](https://raw.githubusercontent.com/AAndyProgram/SCrawler/main/ProgramScreenshots/SettingsGlobalDownloading.png) @@ -123,6 +124,7 @@ - `Store session data` - If checked, session data will be stored in an xml file. The number indicates the number of sessions to be saved. - `Open last mode (users or subscriptions)` - if disabled, the user mode will be used when initializing the feed. - `Show friendly names instead of usernames` +- `Show special feeds in media items` - if you have one or more special feeds, they will appear in the additional post context menu. Disabled by default. **Enabling this setting may increase memory consumption.** ![Feed settings](https://raw.githubusercontent.com/AAndyProgram/SCrawler/main/ProgramScreenshots/SettingsGlobalFeed.png) @@ -132,6 +134,10 @@ If you see something like `Jobs ` in the progress bar, it means that SCr Red highlight means that the field is required or you have entered an incorrect value. White highlight means the field is optional. +**You can add an additional account by clicking the `Add new account` button in the main window.** + +**ALL SETTINGS NOT MARKED IN RED ARE OPTIONAL** + ## Sites requirements - [Reddit](#reddit-requirements) @@ -140,6 +146,7 @@ Red highlight means that the field is required or you have entered an incorrect - [Mastodon](#mastodon-requirements) - [Instagram](#instagram-requirements) - [Threads](#threads-requirements) +- [Facebook](#facebook-requirements) - [JustForFans](#justforfans-requirements) - [RedGifs](#reddit-requirements) - [YouTube](#youtube-requirements) @@ -156,6 +163,7 @@ Red highlight means that the field is required or you have entered an incorrect - `Saved posts path` - if you wish, you can specify a special path for saved posts. Leave it blank to use the default path. *Saved Twitter posts are posts that you have bookmarked.* - `Cookies` add [cookies](#how-to-set-up-cookies) from your browser (click the pencil button to open the cookies editor). - `Download site data` - You can disable downloading data from the site if you need it. If disabled, this site's data will not be downloaded. +- `Download saved posts` - You can enable/disable downloading of saved posts for the editing account (if this site supports downloading of saved posts). - `Get user media only` - the default for creating new users - `Temporary`, `Download images/videos` - personal site settings by default for creating new users. Means the same as the parameters of the same name on the `Defaults` tab. If the checkbox is in an intermediate state, then the same name setting of the `Defaults` tab will be used instead. Otherwise, this setting will be applied. @@ -223,6 +231,8 @@ The Reddit parser can parse data without cookies, but you can add it if you like ### Reddit user settings - `View` - works the same as Reddit view modes (`new`, `hot`, `top`) - `Period` - only works with `Top` view mode and the same as Reddit periods +- `Reddit account` - select the Reddit account that will be used to download the data +- `RedGifs account` - select the RedGifs account that will be used to download the data ![Reddit user settings](https://raw.githubusercontent.com/AAndyProgram/SCrawler/main/ProgramScreenshots/SettingsRedditView.png) @@ -295,9 +305,8 @@ The Reddit parser can parse data without cookies, but you can add it if you like - [`Authorization`](#how-to-find-headers) and [`Token`](#how-to-find-headers) for each domain you want to download from (see [additional settings](#Mastodon-additional-setting)) ### Mastodon additional setting -- `Domains` - select or add a new domain for which you want to add authorization -- `Auth` - Bearer token. Must start with `Bearer ` word followed by a space. -- `Token` - in this field you need to put `x-csrf-token`. + +To support the downloading of this site you should add the Mastodon domain to this list. ![Mastodon additional settings](https://raw.githubusercontent.com/AAndyProgram/SCrawler/main/ProgramScreenshots/SettingsSiteMastodonAdditional.png) @@ -438,6 +447,17 @@ result_cancel-->[*] - [`Cookies`](#how-to-set-up-cookies) - [`headers`](#how-to-find-headers) +## Facebook +- `x-ig-app-id`, `x-asbd-id`, `Accept`, `sec-ch-ua`, `sec-ch-ua-full-version-list`, `sec-ch-ua-platform`, `sec-ch-ua-platform-version`, `UserAgent` - [how to find](#how-to-find-headers) + +The `Accept` and `UserAgent` headers must be obtained from the page `https://facebook.com/`. + +![Facebook settings](https://raw.githubusercontent.com/AAndyProgram/SCrawler/main/ProgramScreenshots/SettingsSiteFacebook.png) + +### Facebook requirements +- [`Cookies`](#how-to-set-up-cookies) +- [`headers`](#how-to-find-headers) + ## JustForFans - `User ID` - read below - `UserHash4` - read below @@ -496,7 +516,7 @@ See additional settings [here](https://github.com/AAndyProgram/SCrawler/wiki/You ### Pinterest requirements - [`Cookies`](#how-to-set-up-cookies) for private data -- [*gallery-dl*](#Gallery-dl) +- [**gallery-dl**](#Gallery-dl) ## TikTok - `Remove tags from title` - if the title contains tags, they will be removed (only works with `Use native title`) @@ -664,7 +684,7 @@ This command can be a batch command or any script file (bat, ps1 or whatever you # How to set up cookies ## First method -1. Using a browser extension to receive site cookies, copy the cookies (or save them to a file). +1. Using a browser extension to receive site cookies, copy the cookies (or save them to a file) (I use this to get cookies: [Official site](https://www.editthiscookie.com/blog/2014/03/install-editthiscookie/), [Chrome Web Store](https://chrome.google.com/webstore/detail/editthiscookie/fngmhnnpilhplaeedifhccceomclgfbg)). 1. Open the settings of the site you want to configure (for example, `Main window` - `Settings` - `Instagram`). 1. Find the `Cookies` field. 1. Click on the `Pencil` button.