mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 16:22:17 +00:00
2023.8.6.0
Plugins.Attributes: add 'DependentFields' attribute Plugins.IPluginContentProvider: add 'Options' and 'IsSubscription' properties Plugins.ISiteSettings: add 'SubscriptionsAllowed' property Plugins.ExchangeOptions: add 'Options' field Plugins.Attributes.PropertyUpdater: replace 'Dependencies' with 'Arguments' YT: add 'OutputPathAskForName' and 'OutputPathAutoAddPaths' properties; add the ability to store download locations; add 'DownloadLocation' and 'DownloadLocationsCollection' objects YT.IDownloaderSettings: add 'OutputPathAskForName' and 'OutputPathAutoAddPaths' properties YT.Downloader: fixed bug with re-saving elements when loading a video list; fixed bug when files were not deleted when clicking on the delete button; fixed a bug that caused the video to redownload; download job removes elements at wrong indexes; added skipping of downloaded elements in the job; fixed a bug, pending option did not change after download complete YT.YouTubeMediaContainerBase: add '_MediaStateOnLoad' field and 'NeedToSave' function; update the 'Save' function to prevent saving a file when a download is complete and the file has already been saved; update code for new yt-dlp version Fixed cache deletion errors Add user queue Add global locations API.Base.SiteSettingsBase: implement 'SubscriptionsAllowed' property; remove request headers with null values on save; add '_AllowUserAgentUpdate' parameter API.Base.Structures: add 'SiteModes' enum API.Base.UserDataBase: add 'Erase' button; implement 'Options' and 'IsSubscription' properties; add 'SpecialLabels' property; update 'LVIKey'; update 'FitToAddParams' function; add 'EraseData' function; user colors; Not UserExists notification, UserQueue support API.Base: add 'DeclaredNames' API.Instagram: remove default values for headers; disable updating UserAgent from global; check for a new username for non-existent users API.Mastodon: bypass new inherited twitter options; update names and headers API.OnlyFans: make 'HH_BROWSER' property nullable; remove 'HH_BROWSER' from required; fix username bug (dots); handling of 504 and 429 errors; add 'DownloadHighlights' and 'DownloadChatMedia' options; add 'UserExchangeOptions'; fixed incorrect error handler API.PathPlugin: fixed incorrect detection of path existence API.Pinterest: add 'SpecialLabels' API.PornHub: add new video regex; remove old regex; added 'DownloadUploaded', 'DownloadTagged', 'DownloadPrivate' and 'DownloadFavorite' properties to 'SiteSettings', 'UserData' and 'UserExchangeOptions'; update regex to define user; added downloading search queries; update 'GetUserUrl' function; hide unnecessary 'RegexFieldsTextBecameNullException' errors; add subscriptions API.Reddit: add 'SpecialLabels'; add bearer token and its refresh interval; add OAuth; add additional options API.RedGifs: add 'DependentFields' for 'Token' API.ThisVid: add 'DownloadFavourite' option; add downloading search queries, tags, categories; add 'SpecialLabels'; add subscriptions; updating cookies issue API.TikTok: rewrite algorithms API.Twitter: add 'UseAppropriateModel', 'UseNewEndPointSearch', 'UseNewEndPointProfiles', 'AbortOnLimit', 'DownloadAlreadyParsed', 'MediaModelAllowNonUserTweets' properties; remove old commented code; remove 'TwitterPic_400' and replace with 'TwitterIcon_32.ToBitmap'; add 'DownloadModelForceApply' user option; update environment to GDL 1.25.8; fixed gifs downloading; fix typo in 'ReparseMissing'; update names API.UserDataBind: prevent adding site-specific labels when adding to a collection API.Xhamster: add downloading search queries, tags, categories; add 'SpecialLabels'; add additional nodes for channels; add subscriptions API.XVIDEOS: add downloading search queries, tags, categories; add 'SpecialLabels'; add subscriptions; changed users creation method; add subscriptions API.YouTube: add subscriptions AutoDownloader: add new group subscription options; update predicates; fixed excluded labels and sites in default mode; update notifications; add an additional skip options, add 'Force start' option DownloadedInfoForm: add subscriptions; fixed size/location bug; hide unnecessary error (refill) Feed: add subscriptions; update filters; add 'Ctrl+G' shortcut FeedMedia: add subscriptions; fixed 'webm' bug; add title for subscription media; add site icon to post; user colors; always using 'FriendlyName' instead of 'UserName' if it exists DownloadGroup, GroupDefaults, GroupParameters: add subscription and 'UsersCount' options MissingPostsForm: add 'BTT_DELETE_ALL' VideoDownloaderForm, DownloaderUrlForm, DownloaderUrlsArrForm: add download locations support VideoDownloaderForm: add subscriptions support GlobalSettingsForm: add new properties UserCreatorForm: add subscriptions; add 'Options' support (of 'ExchangeOptions'); user colors ListImagesLoader: add subscription colors; user colors MainFrame: add subscriptions; add filters by subscription and user; update predicates NuGet: update 'LibVLCSharp', 'LibVLCSharp.WinForms', 'VideoLAN.LibVLC.Windows' DownloadableMediaHost: update 'Save' function PropertyValueHost: fix 'CaptionWidth' bug; add 'Dependents' SettingsHost: add 'Dependents' UserDataHost: add 'Options' and 'IsSubscription' properties SettingsCLS: implement new 'IDownloaderSettings' properties; add 'CacheSnapshots'; add 'DownloadLocations'; add new properties UserInfo, UserFinder: add subscriptions UserSearchForm: fixed search by name bug
This commit is contained in:
Andy
@@ -24,6 +24,8 @@ Namespace API.Twitter
|
||||
ToolTip:="Existing files will be checked for duplicates and duplicates removed." & vbCr &
|
||||
"Works only on the first activation 'Use MD5 comparison'.", LeftOffset:=DefaultOffset)>
|
||||
Friend Property RemoveExistingDuplicates As Boolean = False
|
||||
<PSetting(NameOf(SiteSettings.MediaModelAllowNonUserTweets), NameOf(MySettings), LeftOffset:=DefaultOffset)>
|
||||
Friend Overridable Property MediaModelAllowNonUserTweets As Boolean = False
|
||||
<PSetting(Address:=SettingAddress.User,
|
||||
Caption:="Download model 'Media'",
|
||||
ToolTip:="Download the data using the 'https://twitter.com/UserName/media' command.", LeftOffset:=DefaultOffset)>
|
||||
@@ -36,12 +38,18 @@ Namespace API.Twitter
|
||||
Caption:="Download model 'Search'",
|
||||
ToolTip:="Download the data using the 'https://twitter.com/search?q=from:UserName+include:nativeretweets' command.", LeftOffset:=DefaultOffset)>
|
||||
Friend Overridable Property DownloadModelSearch As Boolean = False
|
||||
<PSetting(Address:=SettingAddress.User,
|
||||
Caption:="Force apply",
|
||||
ToolTip:="Force overrides the default parameters for the first download." & vbCr & "Applies to first download only.", LeftOffset:=DefaultOffset)>
|
||||
Friend Overridable Property DownloadModelForceApply As Boolean = False
|
||||
Private ReadOnly Property MySettings As Object
|
||||
Friend Sub New(ByVal s As SiteSettings)
|
||||
GifsDownload = s.GifsDownload.Value
|
||||
GifsSpecialFolder = s.GifsSpecialFolder.Value
|
||||
GifsPrefix = s.GifsPrefix.Value
|
||||
UseMD5Comparison = s.UseMD5Comparison.Value
|
||||
DownloadModelForceApply = s.UseAppropriateModel.Value
|
||||
MediaModelAllowNonUserTweets = s.MediaModelAllowNonUserTweets.Value
|
||||
MySettings = s
|
||||
End Sub
|
||||
Friend Sub New(ByVal s As Mastodon.SiteSettings)
|
||||
@@ -57,7 +65,9 @@ Namespace API.Twitter
|
||||
GifsPrefix = u.GifsPrefix
|
||||
UseMD5Comparison = u.UseMD5Comparison
|
||||
RemoveExistingDuplicates = u.RemoveExistingDuplicates
|
||||
MediaModelAllowNonUserTweets = u.MediaModelAllowNonUserTweets
|
||||
If Not TypeOf u Is Mastodon.UserData Then
|
||||
DownloadModelForceApply = u.DownloadModelForceApply
|
||||
Dim dm As DModels() = EnumExtract(Of DModels)(u.DownloadModel)
|
||||
If dm.ListExists Then
|
||||
DownloadModelMedia = dm.Contains(DModels.Media)
|
||||
|
||||
@@ -11,50 +11,48 @@ Imports SCrawler.Plugin
|
||||
Imports SCrawler.Plugin.Attributes
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports PersonalUtilities.Tools.Web.Clients
|
||||
Imports DN = SCrawler.API.Base.DeclaredNames
|
||||
Namespace API.Twitter
|
||||
<Manifest(TwitterSiteKey), SavedPosts, SeparatedTasks, SpecialForm(False)>
|
||||
Friend Class SiteSettings : Inherits SiteSettingsBase
|
||||
#Region "Token names"
|
||||
Friend Const Header_Authorization As String = "authorization"
|
||||
Friend Const Header_Token As String = "x-csrf-token"
|
||||
#End Region
|
||||
#Region "Properties constants"
|
||||
Friend Const GifsSpecialFolder_Text As String = "GIFs special folder"
|
||||
Friend Const GifsSpecialFolder_ToolTip As String = "Put the GIFs in a special folder" & vbCr &
|
||||
"This is a folder name, not an absolute path." & vbCr &
|
||||
"This folder(s) will be created relative to the user's root folder." & vbCr &
|
||||
"Examples:" & vbCr & "SomeFolderName" & vbCr & "SomeFolderName\SomeFolderName2"
|
||||
Friend Const GifsPrefix_Text As String = "GIF prefix"
|
||||
Friend Const GifsPrefix_ToolTip As String = "This prefix will be added to the beginning of the filename"
|
||||
Friend Const GifsDownload_Text As String = "Download GIFs"
|
||||
Friend Const UseMD5Comparison_Text As String = "Use MD5 comparison"
|
||||
Friend Const UseMD5Comparison_ToolTip As String = "Each image will be checked for existence using MD5"
|
||||
#End Region
|
||||
#Region "Declarations"
|
||||
Friend Overrides ReadOnly Property Icon As Icon
|
||||
Get
|
||||
Return My.Resources.SiteResources.TwitterIcon_32
|
||||
End Get
|
||||
End Property
|
||||
Private ReadOnly _Image As Image
|
||||
Friend Overrides ReadOnly Property Image As Image
|
||||
Get
|
||||
Return My.Resources.SiteResources.TwitterPic_400
|
||||
Return _Image
|
||||
End Get
|
||||
End Property
|
||||
'TODELETE: twitter headers
|
||||
'#Region "Auth"
|
||||
' <PropertyOption(AllowNull:=False, IsAuth:=False, ControlText:="Authorization",
|
||||
' ControlToolTip:="Set authorization from [authorization] response header. This field must start from [Bearer] key word")>
|
||||
' Private ReadOnly Property Auth As PropertyValue
|
||||
' <PropertyOption(AllowNull:=False, IsAuth:=False, ControlText:="Token", ControlToolTip:="Set token from [x-csrf-token] response header")>
|
||||
' Private ReadOnly Property Token As PropertyValue
|
||||
'#End Region
|
||||
#Region "Other properties"
|
||||
<PropertyOption(ControlText:=GifsDownload_Text), PXML>
|
||||
<PropertyOption(ControlText:="Use the appropriate model",
|
||||
ControlToolTip:="Use the appropriate model for new users." & vbCr &
|
||||
"If disabled, all download models will be used for the first download. " &
|
||||
"Next, the appropriate download model will be automatically selected." & vbCr &
|
||||
"Otherwise the appropriate download model will be selected right from the start."), PXML>
|
||||
Friend ReadOnly Property UseAppropriateModel As PropertyValue
|
||||
#Region "End points"
|
||||
<PropertyOption(ControlText:="New endpoint: search", ControlToolTip:="Use new endpoint argument (-o search-endpoint=graphql) for the search model."), PXML>
|
||||
Friend Property UseNewEndPointSearch As PropertyValue
|
||||
<PropertyOption(ControlText:="New endpoint: profiles", ControlToolTip:="Use new endpoint argument (-o search-endpoint=graphql) for the profile models."), PXML>
|
||||
Friend Property UseNewEndPointProfiles As PropertyValue
|
||||
#End Region
|
||||
#Region "Limits"
|
||||
<PropertyOption(ControlText:="Abort on limit", ControlToolTip:="Abort twitter downloading when limit is reached"), PXML>
|
||||
Friend Property AbortOnLimit As PropertyValue
|
||||
<PropertyOption(ControlText:="Download already parsed", ControlToolTip:="Download already parsed content on abort"), PXML>
|
||||
Friend Property DownloadAlreadyParsed As PropertyValue
|
||||
#End Region
|
||||
<PropertyOption(ControlText:="Media Model: allow non-user tweets", ControlToolTip:="Allow downloading non-user tweets in the media-model."), PXML>
|
||||
Friend ReadOnly Property MediaModelAllowNonUserTweets As PropertyValue
|
||||
<PropertyOption(ControlText:=DN.GifsDownloadCaption), PXML>
|
||||
Friend ReadOnly Property GifsDownload As PropertyValue
|
||||
<PropertyOption(ControlText:=GifsSpecialFolder_Text, ControlToolTip:=GifsSpecialFolder_ToolTip), PXML>
|
||||
<PropertyOption(ControlText:=DN.GifsSpecialFolderCaption, ControlToolTip:=DN.GifsSpecialFolderToolTip), PXML>
|
||||
Friend ReadOnly Property GifsSpecialFolder As PropertyValue
|
||||
<PropertyOption(ControlText:=GifsPrefix_Text, ControlToolTip:=GifsPrefix_ToolTip), PXML>
|
||||
<PropertyOption(ControlText:=DN.GifsPrefixCaption, ControlToolTip:=DN.GifsPrefixToolTip), PXML>
|
||||
Friend ReadOnly Property GifsPrefix As PropertyValue
|
||||
<Provider(NameOf(GifsSpecialFolder), Interaction:=True), Provider(NameOf(GifsPrefix), Interaction:=True)>
|
||||
Private ReadOnly Property GifStringChecker As IFormatProvider
|
||||
@@ -76,52 +74,38 @@ Namespace API.Twitter
|
||||
Throw New NotImplementedException("[GetFormat] is not available in the context of [GifStringProvider]")
|
||||
End Function
|
||||
End Class
|
||||
<PropertyOption(ControlText:=UseMD5Comparison_Text, ControlToolTip:=UseMD5Comparison_ToolTip), PXML>
|
||||
<PropertyOption(ControlText:=DN.UseMD5ComparisonCaption, ControlToolTip:=DN.UseMD5ComparisonToolTip), PXML>
|
||||
Friend ReadOnly Property UseMD5Comparison As PropertyValue
|
||||
<PXML, PropertyOption(ControlText:="Concurrent downloads", ControlToolTip:="The number of concurrent downloads.", LeftOffset:=120), TaskCounter>
|
||||
<PropertyOption(ControlText:=DN.ConcurrentDownloadsCaption,
|
||||
ControlToolTip:=DN.ConcurrentDownloadsToolTip, AllowNull:=False, LeftOffset:=120), PXML, TaskCounter>
|
||||
Friend ReadOnly Property ConcurrentDownloads As PropertyValue
|
||||
<Provider(NameOf(ConcurrentDownloads), FieldsChecker:=True)>
|
||||
Private ReadOnly Property MyConcurrentDownloadsProvider As IFormatProvider
|
||||
#End Region
|
||||
'TODELETE: twitter headers
|
||||
'Private Sub ChangeResponserFields(ByVal PropName As String, ByVal Value As Object)
|
||||
' If Not PropName.IsEmptyString Then
|
||||
' Dim f$ = String.Empty
|
||||
' Select Case PropName
|
||||
' Case NameOf(Auth) : f = Header_Authorization
|
||||
' Case NameOf(Token) : f = Header_Token
|
||||
' End Select
|
||||
' If Not f.IsEmptyString Then
|
||||
' Responser.Headers.Remove(f)
|
||||
' If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
|
||||
' Responser.SaveSettings()
|
||||
' End If
|
||||
' End If
|
||||
'End Sub
|
||||
#End Region
|
||||
Friend Sub New()
|
||||
MyBase.New(TwitterSite, "twitter.com")
|
||||
|
||||
'TODELETE: twitter headers
|
||||
'Dim a$ = String.Empty
|
||||
'Dim t$ = String.Empty
|
||||
_Image = My.Resources.SiteResources.TwitterIcon_32.ToBitmap
|
||||
|
||||
With Responser
|
||||
'TODELETE: twitter headers
|
||||
'a = .Headers.Value(Header_Authorization)
|
||||
't = .Headers.Value(Header_Token)
|
||||
.Cookies.ChangedAllowInternalDrop = False
|
||||
.Cookies.Changed = False
|
||||
End With
|
||||
|
||||
'TODELETE: twitter headers
|
||||
'Auth = New PropertyValue(a, GetType(String), Sub(v) ChangeResponserFields(NameOf(Auth), v))
|
||||
'Token = New PropertyValue(t, GetType(String), Sub(v) ChangeResponserFields(NameOf(Token), v))
|
||||
|
||||
UseAppropriateModel = New PropertyValue(True)
|
||||
UseNewEndPointSearch = New PropertyValue(True)
|
||||
UseNewEndPointProfiles = New PropertyValue(True)
|
||||
AbortOnLimit = New PropertyValue(True)
|
||||
DownloadAlreadyParsed = New PropertyValue(True)
|
||||
MediaModelAllowNonUserTweets = New PropertyValue(False)
|
||||
GifsDownload = New PropertyValue(True)
|
||||
GifsSpecialFolder = New PropertyValue(String.Empty, GetType(String))
|
||||
GifsPrefix = New PropertyValue("GIF_")
|
||||
GifStringChecker = New GifStringProvider
|
||||
UseMD5Comparison = New PropertyValue(False)
|
||||
ConcurrentDownloads = New PropertyValue(1)
|
||||
MyConcurrentDownloadsProvider = New ConcurrentDownloadsProvider
|
||||
|
||||
UserRegex = RParams.DMS("[htps:/]{7,8}.*?twitter.com/([^/]+)", 1)
|
||||
UrlPatternUser = "https://twitter.com/{0}"
|
||||
@@ -141,6 +125,11 @@ Namespace API.Twitter
|
||||
Friend Overrides Function Available(ByVal What As ISiteSettings.Download, ByVal Silent As Boolean) As Boolean
|
||||
Return Settings.GalleryDLFile.Exists And BaseAuthExists()
|
||||
End Function
|
||||
Friend Property LIMIT_ABORT As Boolean = False
|
||||
Friend Overrides Sub DownloadDone(ByVal What As ISiteSettings.Download)
|
||||
LIMIT_ABORT = False
|
||||
MyBase.DownloadDone(What)
|
||||
End Sub
|
||||
Friend Overrides Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean)
|
||||
If Options Is Nothing OrElse (Not TypeOf Options Is EditorExchangeOptions OrElse
|
||||
Not DirectCast(Options, EditorExchangeOptions).SiteKey = TwitterSiteKey) Then _
|
||||
|
||||
@@ -20,6 +20,8 @@ Namespace API.Twitter
|
||||
#Region "XML names"
|
||||
Private Const Name_FirstDownloadComplete As String = "FirstDownloadComplete"
|
||||
Private Const Name_DownloadModel As String = "DownloadModel"
|
||||
Private Const Name_DownloadModelForceApply As String = "DownloadModelForceApply"
|
||||
Private Const Name_MediaModelAllowNonUserTweets As String = "MediaModelAllowNonUserTweets"
|
||||
Private Const Name_GifsDownload As String = "GifsDownload"
|
||||
Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder"
|
||||
Private Const Name_GifsPrefix As String = "GifsPrefix"
|
||||
@@ -32,7 +34,9 @@ Namespace API.Twitter
|
||||
Search = 5
|
||||
End Enum
|
||||
Private FirstDownloadComplete As Boolean = False
|
||||
Friend Property DownloadModelForceApply As Boolean = False
|
||||
Friend Property DownloadModel As DownloadModels = DownloadModels.Undefined
|
||||
Friend Property MediaModelAllowNonUserTweets As Boolean = False
|
||||
Friend Property GifsDownload As Boolean = True
|
||||
Friend Property GifsSpecialFolder As String = String.Empty
|
||||
Friend Property GifsPrefix As String = String.Empty
|
||||
@@ -64,6 +68,8 @@ Namespace API.Twitter
|
||||
UseMD5Comparison = .UseMD5Comparison
|
||||
RemoveExistingDuplicates = .RemoveExistingDuplicates
|
||||
DownloadModel = DownloadModels.Undefined
|
||||
DownloadModelForceApply = .DownloadModelForceApply
|
||||
MediaModelAllowNonUserTweets = .MediaModelAllowNonUserTweets
|
||||
If .DownloadModelMedia Then DownloadModel += DownloadModels.Media
|
||||
If .DownloadModelProfile Then DownloadModel += DownloadModels.Profile
|
||||
If .DownloadModelSearch Then DownloadModel += DownloadModels.Search
|
||||
@@ -78,6 +84,7 @@ Namespace API.Twitter
|
||||
Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
|
||||
With Container
|
||||
If Loading Then
|
||||
DownloadModelForceApply = .Value(Name_DownloadModelForceApply).FromXML(Of Boolean)(False)
|
||||
If .Contains(Name_FirstDownloadComplete) Then
|
||||
FirstDownloadComplete = .Value(Name_FirstDownloadComplete).FromXML(Of Boolean)(False)
|
||||
DownloadModel = .Value(Name_DownloadModel).FromXML(Of Integer)(DownloadModels.Undefined)
|
||||
@@ -87,6 +94,7 @@ Namespace API.Twitter
|
||||
DownloadModel = .Value(Name_DownloadModel).FromXML(Of Integer)(DownloadModels.Undefined)
|
||||
Else
|
||||
If FirstDownloadComplete Then
|
||||
DownloadModelForceApply = False
|
||||
If ParseUserMediaOnly Then
|
||||
DownloadModel = DownloadModels.Media
|
||||
Else
|
||||
@@ -107,8 +115,10 @@ Namespace API.Twitter
|
||||
UseMD5Comparison = .Value(Name_UseMD5Comparison).FromXML(Of Boolean)(False)
|
||||
RemoveExistingDuplicates = .Value(Name_RemoveExistingDuplicates).FromXML(Of Boolean)(False)
|
||||
StartMD5Checked = .Value(Name_StartMD5Checked).FromXML(Of Boolean)(False)
|
||||
MediaModelAllowNonUserTweets = .Value(Name_MediaModelAllowNonUserTweets).FromXML(Of Boolean)(False)
|
||||
Else
|
||||
.Add(Name_FirstDownloadComplete, FirstDownloadComplete.BoolToInteger)
|
||||
.Add(Name_DownloadModelForceApply, DownloadModelForceApply.BoolToInteger)
|
||||
.Add(Name_DownloadModel, CInt(DownloadModel))
|
||||
.Add(Name_GifsDownload, GifsDownload.BoolToInteger)
|
||||
.Add(Name_GifsSpecialFolder, GifsSpecialFolder)
|
||||
@@ -116,18 +126,29 @@ Namespace API.Twitter
|
||||
.Add(Name_UseMD5Comparison, UseMD5Comparison.BoolToInteger)
|
||||
.Add(Name_RemoveExistingDuplicates, RemoveExistingDuplicates.BoolToInteger)
|
||||
.Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger)
|
||||
.Add(Name_MediaModelAllowNonUserTweets, MediaModelAllowNonUserTweets.BoolToInteger)
|
||||
End If
|
||||
End With
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download functions"
|
||||
Private Function GetContainerSubnodes() As List(Of String())
|
||||
Return New List(Of String()) From {
|
||||
{{"content", "itemContent", "tweet_results", "result", "legacy"}},
|
||||
{{"content", "itemContent", "tweet_results", "result", "tweet", "legacy"}}
|
||||
}
|
||||
End Function
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
If IsSavedPosts Then
|
||||
If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.Post.ID), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
|
||||
DownloadData_SavedPosts(Token)
|
||||
If MySettings.LIMIT_ABORT Then
|
||||
TwitterLimitException.LogMessage(ToStringForLog, True)
|
||||
Else
|
||||
If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.File.File), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
|
||||
DownloadData_Timeline(Token)
|
||||
If IsSavedPosts Then
|
||||
If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.Post.ID), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
|
||||
DownloadData_SavedPosts(Token)
|
||||
Else
|
||||
If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.File.File), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
|
||||
DownloadData_Timeline(Token)
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
Private Sub DownloadData_Timeline(ByVal Token As CancellationToken)
|
||||
@@ -139,6 +160,8 @@ Namespace API.Twitter
|
||||
Dim PostDate$, tmpUserId$
|
||||
Dim i%
|
||||
Dim dirIndx% = -1
|
||||
Dim nodes As List(Of String()) = GetContainerSubnodes()
|
||||
Dim node$()
|
||||
Dim timelineNode As Predicate(Of EContainer) = Function(ee) ee.Value("type").StringToLower = "timelineaddentries"
|
||||
Dim pinNode As Predicate(Of EContainer) = Function(ee) ee.Value("type").StringToLower = "timelinepinentry"
|
||||
Dim entriesNode As Predicate(Of EContainer) = Function(ee) ee.Name = "entries" Or ee.Name = entry
|
||||
@@ -150,13 +173,15 @@ Namespace API.Twitter
|
||||
|
||||
Dim __parseContainer As Func(Of EContainer, Boolean) =
|
||||
Function(ByVal ee As EContainer) As Boolean
|
||||
If dirIndx <= 1 Then
|
||||
nn = ee({"content", "itemContent", "tweet_results", "result", "legacy"})
|
||||
Else
|
||||
nn = ee
|
||||
nn = Nothing
|
||||
If dirIndx > 1 Then nn = ee
|
||||
If Not nn.ListExists Then
|
||||
For Each node In nodes
|
||||
nn = ee(node)
|
||||
If nn.ListExists Then Exit For
|
||||
Next
|
||||
End If
|
||||
|
||||
If Not nn.ListExists Then nn = ee({"content", "itemContent", "tweet_results", "result", "tweet", "legacy"})
|
||||
If nn.ListExists Then
|
||||
PostID = nn.Value("id_str").IfNullOrEmpty(nn.Value("id"))
|
||||
|
||||
@@ -181,15 +206,14 @@ Namespace API.Twitter
|
||||
If tmpUserId.IsEmptyString Then tmpUserId = nn.ItemF({"extended_entities", "media", 0, sourceIdPredicate}).XmlIfNothingValue.
|
||||
IfNullOrEmpty(nn.Value("user_id")).IfNullOrEmpty(nn.Value("user_id_str")).IfNullOrEmpty("/")
|
||||
|
||||
If Not ParseUserMediaOnly OrElse (Not ID.IsEmptyString AndAlso tmpUserId = ID) Then ObtainMedia(nn, PostID, PostDate)
|
||||
If Not ParseUserMediaOnly OrElse
|
||||
(dirIndx = 0 AndAlso MediaModelAllowNonUserTweets) OrElse
|
||||
(Not ID.IsEmptyString AndAlso tmpUserId = ID) Then ObtainMedia(nn, PostID, PostDate)
|
||||
End If
|
||||
Return True
|
||||
End Function
|
||||
|
||||
tCache = New CacheKeeper($"{DownloadContentDefault_GetRootDir()}\_tCache\") With {
|
||||
.CacheDeleteError = New ErrorsDescriber(EDP.None) With {.Action = Sub(ee, eex, msg, obj) Settings.Cache.AddPath(tCache)}}
|
||||
If tCache.RootDirectory.Exists(SFO.Path, False) Then tCache.RootDirectory.Delete(SFO.Path, SFODelete.DeletePermanently, EDP.ReturnValue)
|
||||
tCache.Validate()
|
||||
tCache = CreateCache()
|
||||
|
||||
Dim dirs As List(Of SFile) = GetTimelineFromGalleryDL(tCache, Token)
|
||||
If dirs.ListExists Then
|
||||
@@ -313,7 +337,9 @@ Namespace API.Twitter
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
DownloadModelForceApply = False
|
||||
FirstDownloadComplete = True
|
||||
Catch limit_ex As TwitterLimitException
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"data downloading error [{URL}]")
|
||||
Finally
|
||||
@@ -328,6 +354,8 @@ Namespace API.Twitter
|
||||
If files.ListExists Then
|
||||
ResetFileNameProvider(Math.Max(files.Count.ToString.Length, 3))
|
||||
Dim id$
|
||||
Dim nodes As List(Of String()) = GetContainerSubnodes()
|
||||
Dim node$()
|
||||
Dim j As EContainer, jj As EContainer
|
||||
Dim jErr As New ErrorsDescriber(EDP.ReturnValue)
|
||||
For i% = 0 To files.Count - 1
|
||||
@@ -339,19 +367,24 @@ Namespace API.Twitter
|
||||
ProgressPre.ChangeMax(.Count)
|
||||
For Each jj In .Self
|
||||
ProgressPre.Perform()
|
||||
With jj({"content", "itemContent", "tweet_results", "result", "legacy"})
|
||||
If .ListExists Then
|
||||
id = .Value("id_str")
|
||||
If _TempPostsList.Contains(id) Then j.Dispose() : Exit Sub Else ObtainMedia(.Self, id, .Value("created_at"))
|
||||
End If
|
||||
End With
|
||||
For Each node In nodes
|
||||
With jj(node)
|
||||
If .ListExists Then
|
||||
id = .Value("id_str")
|
||||
If _TempPostsList.Contains(id) Then j.Dispose() : Exit Sub Else ObtainMedia(.Self, id, .Value("created_at"))
|
||||
Exit For
|
||||
End If
|
||||
End With
|
||||
Next
|
||||
Next
|
||||
End If
|
||||
End With
|
||||
j.Dispose()
|
||||
End If
|
||||
Next
|
||||
nodes.Clear()
|
||||
End If
|
||||
Catch limit_ex As TwitterLimitException
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, "data downloading error (Saved Posts)")
|
||||
End Try
|
||||
@@ -408,30 +441,28 @@ Namespace API.Twitter
|
||||
Dim f As SFile
|
||||
Dim m As UserMedia
|
||||
If w.ListExists Then
|
||||
For Each n As EContainer In w
|
||||
If n.Value("type") = "animated_gif" Then
|
||||
With n({"video_info", "variants"})
|
||||
If .ListExists Then
|
||||
With .ItemF({gifUrl})
|
||||
If .ListExists Then
|
||||
url = .Value("url")
|
||||
ff = UrlFile(url)
|
||||
If Not ff.IsEmptyString Then
|
||||
If GifsDownload And Not _DataNames.Contains(ff) Then
|
||||
m = MediaFromData(url, PostID, PostDate,, State, UTypes.Video)
|
||||
f = m.File
|
||||
If Not f.IsEmptyString And Not GifsPrefix.IsEmptyString Then f.Name = $"{GifsPrefix}{f.Name}" : m.File = f
|
||||
If Not GifsSpecialFolder.IsEmptyString Then m.SpecialFolder = $"{GifsSpecialFolder}*"
|
||||
_TempMediaList.ListAddValue(m, LNC)
|
||||
End If
|
||||
Return True
|
||||
If w.Value("type") = "animated_gif" Then
|
||||
With w({"video_info", "variants"})
|
||||
If .ListExists Then
|
||||
With .ItemF({gifUrl})
|
||||
If .ListExists Then
|
||||
url = .Value("url")
|
||||
ff = UrlFile(url)
|
||||
If Not ff.IsEmptyString Then
|
||||
If GifsDownload And Not _DataNames.Contains(ff) Then
|
||||
m = MediaFromData(url, PostID, PostDate,, State, UTypes.Video)
|
||||
f = m.File
|
||||
If Not f.IsEmptyString And Not GifsPrefix.IsEmptyString Then f.Name = $"{GifsPrefix}{f.Name}" : m.File = f
|
||||
If Not GifsSpecialFolder.IsEmptyString Then m.SpecialFolder = $"{GifsSpecialFolder}*"
|
||||
_TempMediaList.ListAddValue(m, LNC)
|
||||
End If
|
||||
Return True
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End If
|
||||
Return False
|
||||
Catch ex As Exception
|
||||
@@ -460,13 +491,22 @@ Namespace API.Twitter
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Gallery-DL Support"
|
||||
Private Class TwitterLimitException : Inherits Exception
|
||||
Friend Sub New(ByVal User As String, ByVal Skipped As Boolean)
|
||||
LogMessage(User, Skipped)
|
||||
End Sub
|
||||
Friend Shared Sub LogMessage(ByVal User As String, ByVal Skipped As Boolean)
|
||||
MyMainLOG = $"{User}: twitter limit reached.{IIf(Skipped, "Data has not been downloaded", String.Empty)}"
|
||||
End Sub
|
||||
End Class
|
||||
Private Class TwitterGDL : Inherits GDL.GDLBatch
|
||||
Private Property Token As CancellationToken
|
||||
Friend Sub New(ByVal Dir As SFile, ByVal _Token As CancellationToken)
|
||||
MyBase.New
|
||||
Private ReadOnly KillOnLimit As Boolean
|
||||
Friend LimitReached As Boolean = False
|
||||
Friend Sub New(ByVal Dir As SFile, ByVal _Token As CancellationToken, ByVal _KillOnLimit As Boolean)
|
||||
MyBase.New(_Token)
|
||||
Commands.Clear()
|
||||
If Not Dir.IsEmptyString Then ChangeDirectory(Dir)
|
||||
Token = _Token
|
||||
KillOnLimit = _KillOnLimit
|
||||
End Sub
|
||||
Protected Overrides Async Function Validate(ByVal Value As String) As Task
|
||||
If Not ProcessKilled AndAlso Await Task.Run(Function() Token.IsCancellationRequested OrElse IdExists(Value)) Then Kill()
|
||||
@@ -482,14 +522,27 @@ Namespace API.Twitter
|
||||
End Try
|
||||
Return False
|
||||
End Function
|
||||
Protected Overrides Async Sub ErrorDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
|
||||
Await Task.Run(Sub() CheckForLimit(e.Data))
|
||||
End Sub
|
||||
Private Sub CheckForLimit(ByVal Value As String)
|
||||
If Token.IsCancellationRequested Or (KillOnLimit AndAlso Not ProcessKilled AndAlso
|
||||
Not Value.IsEmptyString AndAlso Value.ToLower.Contains("for rate limit reset")) Then
|
||||
LimitReached = True
|
||||
Kill()
|
||||
End If
|
||||
End Sub
|
||||
End Class
|
||||
Private Function GetDataFromGalleryDL(ByVal URL As String, ByVal Cache As CacheKeeper, ByVal UseTempPostList As Boolean,
|
||||
Optional ByVal Token As CancellationToken = Nothing) As SFile
|
||||
Dim command$ = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --cookies ""{MySettings.CookiesNetscapeFile}"" --write-pages "
|
||||
Dim command$ = String.Empty
|
||||
Try
|
||||
Dim conf As SFile = GdlCreateConf(Cache.NewPath)
|
||||
command = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --config ""{conf}"" --write-pages "
|
||||
command &= GdlGetIdFilterString()
|
||||
Dim dir As SFile = Cache.NewPath
|
||||
If dir.Exists(SFO.Path,, EDP.ThrowException) Then
|
||||
Using batch As New TwitterGDL(dir, Token)
|
||||
Using batch As New TwitterGDL(dir, Token, MySettings.AbortOnLimit.Value)
|
||||
If UseTempPostList Then
|
||||
batch.TempPostsList = _TempPostsList
|
||||
command &= GdlGetIdFilterString()
|
||||
@@ -499,10 +552,22 @@ Namespace API.Twitter
|
||||
'Debug.WriteLine(command)
|
||||
'#End If
|
||||
batch.Execute(command)
|
||||
If batch.LimitReached Then
|
||||
If CBool(MySettings.DownloadAlreadyParsed.Value) And
|
||||
SFile.GetFiles(dir, "*.txt", IO.SearchOption.AllDirectories, EDP.ReturnValue).Count > 0 Then
|
||||
MySettings.LIMIT_ABORT = True
|
||||
Return dir
|
||||
Else
|
||||
Throw New TwitterLimitException(ToStringForLog, False)
|
||||
End If
|
||||
End If
|
||||
End Using
|
||||
Return dir
|
||||
End If
|
||||
Return Nothing
|
||||
Catch limit_ex As TwitterLimitException
|
||||
MySettings.LIMIT_ABORT = True
|
||||
Throw limit_ex
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetDataFromGalleryDL({command})")
|
||||
End Try
|
||||
@@ -511,20 +576,23 @@ Namespace API.Twitter
|
||||
Dim command$ = String.Empty
|
||||
Try
|
||||
Dim confCache As CacheKeeper = Cache.NewInstance(Of BatchFileExchanger)
|
||||
Dim conf As SFile = $"{confCache.RootDirectory.PathWithSeparator}TwitterGdlConfig.conf"
|
||||
Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") &
|
||||
""",""cookies-update"": false,""twitter"":{""cards"": false,""conversations"": true,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}"
|
||||
If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf)
|
||||
If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf)
|
||||
Dim conf As SFile = GdlCreateConf(confCache.RootDirectory)
|
||||
|
||||
If DownloadModel = DownloadModels.Undefined And Not FirstDownloadComplete And DownloadModelForceApply Then
|
||||
If ParseUserMediaOnly Then
|
||||
DownloadModel = DownloadModels.Media
|
||||
Else
|
||||
DownloadModel = DownloadModels.Media + DownloadModels.Profile + DownloadModels.Search
|
||||
End If
|
||||
End If
|
||||
|
||||
Dim outList As New List(Of SFile)
|
||||
Dim rootDir As CacheKeeper = Cache.NewInstance
|
||||
Dim dir As SFile
|
||||
Dim dm As List(Of DownloadModels) = EnumExtract(Of DownloadModels)(DownloadModel).ListIfNothing
|
||||
Dim process As Boolean
|
||||
Dim bProcess As Boolean = DownloadModel = DownloadModels.Undefined Or Not FirstDownloadComplete
|
||||
|
||||
Using tgdl As New TwitterGDL(Nothing, Token) With {
|
||||
Using tgdl As New TwitterGDL(Nothing, Token, MySettings.AbortOnLimit.Value) With {
|
||||
.TempPostsList = _TempPostsList,
|
||||
.AutoClear = True,
|
||||
.AutoReset = True,
|
||||
@@ -541,22 +609,36 @@ Namespace API.Twitter
|
||||
command = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --config ""{conf}"" --write-pages "
|
||||
command &= GdlGetIdFilterString()
|
||||
Select Case i
|
||||
Case 0 : command &= $"https://twitter.com/{Name}/media" : process = bProcess Or dm.Contains(DownloadModels.Media)
|
||||
Case 1 : command &= $"https://twitter.com/{Name}" : process = bProcess Or dm.Contains(DownloadModels.Profile)
|
||||
Case 2 : command &= $"https://twitter.com/search?q=from:{Name}+include:nativeretweets" : process = bProcess Or dm.Contains(DownloadModels.Search)
|
||||
Case 0 : command &= $"https://twitter.com/{Name}/media" : process = dm.Contains(DownloadModels.Media)
|
||||
Case 1 : command &= $"https://twitter.com/{Name}" : process = dm.Contains(DownloadModels.Profile)
|
||||
Case 2 : command &= $"-o search-endpoint=graphql https://twitter.com/search?q=from:{Name}+include:nativeretweets" : process = dm.Contains(DownloadModels.Search)
|
||||
Case Else : process = False
|
||||
End Select
|
||||
'#If DEBUG Then
|
||||
'Debug.WriteLine(command)
|
||||
'#End If
|
||||
ThrowAny(Token)
|
||||
If process Then tgdl.Execute(command)
|
||||
If process Then
|
||||
tgdl.Execute(command)
|
||||
If tgdl.LimitReached Then
|
||||
If CBool(MySettings.DownloadAlreadyParsed.Value) And
|
||||
SFile.GetFiles(rootDir, "*.txt", IO.SearchOption.AllDirectories, EDP.ReturnValue).Count > 0 Then
|
||||
MySettings.LIMIT_ABORT = True
|
||||
Exit For
|
||||
Else
|
||||
Throw New TwitterLimitException(ToStringForLog, False)
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
ThrowAny(Token)
|
||||
Next
|
||||
End Using
|
||||
dm.Clear()
|
||||
|
||||
Return outList
|
||||
Catch limit_ex As TwitterLimitException
|
||||
MySettings.LIMIT_ABORT = True
|
||||
Throw limit_ex
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"{ToStringForLog()}: GetTimelineFromGalleryDL({command})")
|
||||
Return Nothing
|
||||
@@ -565,6 +647,20 @@ Namespace API.Twitter
|
||||
Private Function GdlGetIdFilterString() As String
|
||||
Return If(_TempPostsList.Count > 0, $"--filter ""int(tweet_id) > {_TempPostsList.Last} or abort()"" ", String.Empty)
|
||||
End Function
|
||||
Private Function GdlCreateConf(ByVal Path As SFile) As SFile
|
||||
Try
|
||||
Dim conf As SFile = $"{Path.PathWithSeparator}TwitterGdlConfig.conf"
|
||||
Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") &
|
||||
""",""cookies-update"": false,""twitter"":{""tweet-endpoint"": ""detail"",""cards"": false,""conversations"": true,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}"
|
||||
If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf)
|
||||
If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf)
|
||||
Return conf
|
||||
Catch file_ex As IO.FileNotFoundException
|
||||
Throw file_ex
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "gallery-dl configuration file creating error", New SFile)
|
||||
End Try
|
||||
End Function
|
||||
#End Region
|
||||
#Region "ReparseMissing"
|
||||
Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
|
||||
@@ -576,7 +672,9 @@ Namespace API.Twitter
|
||||
If ContentMissingExists Then
|
||||
Dim m As UserMedia
|
||||
Dim PostDate$
|
||||
Dim j As EContainer
|
||||
Dim nodes As List(Of String()) = GetContainerSubnodes()
|
||||
Dim node$()
|
||||
Dim j As EContainer, n As EContainer
|
||||
Dim f As SFile
|
||||
Dim i%, ii%
|
||||
Dim files As List(Of SFile)
|
||||
@@ -585,6 +683,7 @@ Namespace API.Twitter
|
||||
cache = Settings.Cache
|
||||
Else
|
||||
cache = New CacheKeeper(DownloadContentDefault_GetRootDir.CSFilePS)
|
||||
cache.CacheDeleteError = CacheDeletionError(cache)
|
||||
End If
|
||||
ProgressPre.ChangeMax(_ContentList.Count)
|
||||
For i = 0 To _ContentList.Count - 1
|
||||
@@ -598,7 +697,7 @@ Namespace API.Twitter
|
||||
Else
|
||||
URL = String.Format(SinglePostPattern, Name, m.Post.ID)
|
||||
End If
|
||||
f = GetDataFromGalleryDL(URL, cache, Favorite, Token)
|
||||
f = GetDataFromGalleryDL(URL, cache, False, Token)
|
||||
If Not f.IsEmptyString Then
|
||||
files = SFile.GetFiles(f, "*.txt")
|
||||
If files.ListExists Then
|
||||
@@ -606,13 +705,20 @@ Namespace API.Twitter
|
||||
f = RenameGdlFile(files(ii), ii)
|
||||
j = JsonDocument.Parse(f.GetText)
|
||||
If Not j Is Nothing Then
|
||||
With j.ItemF({"data", 0, "instructions", 0, "entries", 0,
|
||||
"content", "itemContent", "tweet_results", "result", "legacy"})
|
||||
With j.ItemF({"data", 0, "instructions", 0, "entries"})
|
||||
If .ListExists Then
|
||||
PostDate = String.Empty
|
||||
If .Contains("created_at") Then PostDate = .Value("created_at") Else PostDate = String.Empty
|
||||
ObtainMedia(.Self, m.Post.ID, PostDate, UStates.Missing)
|
||||
rList.Add(i)
|
||||
For Each n In .Self
|
||||
For Each node In nodes
|
||||
With n(node)
|
||||
If .ListExists Then
|
||||
PostDate = String.Empty
|
||||
If .Contains("created_at") Then PostDate = .Value("created_at") Else PostDate = String.Empty
|
||||
ObtainMedia(.Self, m.Post.ID, PostDate, UStates.Missing)
|
||||
rList.ListAddValue(i, LNC)
|
||||
End If
|
||||
End With
|
||||
Next
|
||||
Next
|
||||
End If
|
||||
End With
|
||||
j.Dispose()
|
||||
@@ -630,7 +736,7 @@ Namespace API.Twitter
|
||||
Finally
|
||||
If Not cache Is Nothing And Not IsSingleObjectDownload Then cache.Dispose()
|
||||
If rList.Count > 0 Then
|
||||
For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(i) : Next
|
||||
For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(rList(i)) : Next
|
||||
rList.Clear()
|
||||
End If
|
||||
End Try
|
||||
|
||||
Reference in New Issue
Block a user