From be97752b7cdf1dc0e66366a08bae0e8594e3f66a Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:21:04 +0300 Subject: [PATCH] 2026.3.23.0 API.EditorExchangeOptionsBase: move the 'Apply' function API.EditorExchangeOptionsBase_P: remove the 'Apply' function API.UserDataBase: update the 'UserSiteNameUpdate' function; remove the '_ForceSaveUserInfoOnException' property; update the 'DownloadContentDefault_ConvertWebp' function; add the 'DownloadContentDefault_ConvertWebp_TestImg' and 'DownloadContentDefault_ConvertWebp_Impl' functions API.Bluesky: add the 'EditorExchangeOptions' class; add 'did:' handle for recognizing API.PornHub: update M3U8 download functions API.TikTok: fix where gallery-dl would hang when downloading stories API.Twitter: fix downloading a single media file (standalone downloader) fails API.Xhamster: add 'webm' to 'jpg' correct conversion FeedMedia: move the 'ImageRenderer2' class into the 'UserImage' class UserImage: add the 'ImageRenderer2' class --- Changelog.md | 25 +++++++- .../API/Base/EditorExchangeOptionsBase.vb | 4 ++ .../API/Base/EditorExchangeOptionsBase_P.vb | 4 -- SCrawler/API/Base/UserDataBase.vb | 39 +++++++++--- SCrawler/API/Bluesky/EditorExchangeOptions.vb | 44 ++++++++++++++ SCrawler/API/Bluesky/SiteSettings.vb | 14 ++++- SCrawler/API/Bluesky/UserData.vb | 59 +++++++++++++++++-- SCrawler/API/Instagram/UserData.vb | 2 +- SCrawler/API/PornHub/M3U8.vb | 2 +- SCrawler/API/PornHub/UserData.vb | 22 +++---- SCrawler/API/PornHub/UserExchangeOptions.vb | 2 +- SCrawler/API/ThisVid/UserExchangeOptions.vb | 2 +- SCrawler/API/ThreadsNet/UserData.vb | 2 - SCrawler/API/TikTok/UserData.vb | 20 +++++-- SCrawler/API/Twitter/UserData.vb | 12 ++-- SCrawler/API/Xhamster/UserData.vb | 4 ++ SCrawler/API/Xhamster/UserExchangeOptions.vb | 2 +- SCrawler/Download/Feed/FeedMedia.vb | 37 +----------- SCrawler/My Project/AssemblyInfo.vb | 4 +- SCrawler/SCrawler.vbproj | 1 + SCrawler/UserImage.vb | 41 +++++++++++++ 21 files changed, 253 insertions(+), 89 deletions(-) create mode 100644 SCrawler/API/Bluesky/EditorExchangeOptions.vb diff --git a/Changelog.md b/Changelog.md index db0a628..514cc93 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,13 +2,34 @@ - [ffmpeg](https://github.com/AAndyProgram/SCrawler/wiki/Settings#ffmpeg) - x64 version - [release](https://github.com/GyanD/codexffmpeg/releases/tag/5.1.2); [zip](https://github.com/GyanD/codexffmpeg/releases/download/5.1.2/ffmpeg-5.1.2-full_build.zip); **version `5.1.2-full_build-www.gyan.dev`** - x86 version - [release](https://github.com/yt-dlp/FFmpeg-Builds/releases/tag/autobuild-2022-11-30-12-57); [zip](https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-2022-11-30-12-57/ffmpeg-N-109274-gd7a5f068c2-win32-gpl.zip); **version `N-109457-geeb280f351-20221226`** -- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.6** -- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.02.04.233607** +- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.9** +- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.03.17** - [Deno](https://github.com/AAndyProgram/SCrawler/wiki/Settings#deno) - latest *(`2.0.0` or higher)* - [OF-Scraper](https://github.com/AAndyProgram/SCrawler/wiki/Settings#of-scraper) - **3.12.9** ([release](https://github.com/datawhores/OF-Scraper/releases/tag/3.12.9)) # 2026 +## 2026.3.23.0 + +*2026-03-23* + +- Added + - Sites: + - Bluesky: + - download models (user options), like in Twitter + - `did` handle for recognizing + - PornHub: `M3U8` download functionality update + - xHamster: add `webm` to `jpg` correct conversion + - Minor improvements +- Updated + - gallery-dl up to version **1.31.9** + - yt-dlp up to version **2026.03.17** +- Fixed + - Sites: + - TikTok: gallery-dl downloads stories slowly when the user has no stories + - Twitter: downloading a single media file (standalone downloader) fails + - Minor bugs + ## 2026.2.14.0 *2026-02-14* diff --git a/SCrawler/API/Base/EditorExchangeOptionsBase.vb b/SCrawler/API/Base/EditorExchangeOptionsBase.vb index 257cc66..597fb5f 100644 --- a/SCrawler/API/Base/EditorExchangeOptionsBase.vb +++ b/SCrawler/API/Base/EditorExchangeOptionsBase.vb @@ -42,5 +42,9 @@ Namespace API.Base u.DownloadTextSpecialFolder = DownloadTextSpecialFolder End If End Sub + Friend Overridable Sub Apply(ByRef u As UserDataBase) + ApplyBase(u) + If TypeOf u Is IPSite And TypeOf Me Is IPSite Then DirectCast(u, IPSite).QueryString = DirectCast(Me, IPSite).QueryString + End Sub End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/Base/EditorExchangeOptionsBase_P.vb b/SCrawler/API/Base/EditorExchangeOptionsBase_P.vb index a98bb01..a97cf76 100644 --- a/SCrawler/API/Base/EditorExchangeOptionsBase_P.vb +++ b/SCrawler/API/Base/EditorExchangeOptionsBase_P.vb @@ -31,10 +31,6 @@ Namespace API.Base MyBase.New(s) DisableBase() End Sub - Friend Overridable Sub Apply(ByRef u As IPSite) - ApplyBase(u) - u.QueryString = QueryString - End Sub Protected Overridable Sub DisableBase() _ApplyBase_Name = False _ApplyBase_Text = False diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index 95b2614..b7541f5 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -360,8 +360,8 @@ Namespace API.Base Me._UserSiteName = _UserSiteName End Set End Property - Protected Sub UserSiteNameUpdate(ByVal NewName As String) - If Not NewName.IsEmptyString And (UserSiteName.IsEmptyString Or Settings.UpdateUserSiteNameEveryTime) Then UserSiteName = NewName + Protected Sub UserSiteNameUpdate(ByVal NewName As String, Optional ByVal Force As Boolean = False) + If Not NewName.IsEmptyString And (UserSiteName.IsEmptyString Or Settings.UpdateUserSiteNameEveryTime Or Force) Then UserSiteName = NewName End Sub Friend ReadOnly Property UserModel As UsageModel Implements IUserData.UserModel Get @@ -1030,7 +1030,7 @@ BlockNullPicture: End Try End Sub Private Sub UpdateUserInformation_Ex() - If _ForceSaveUserInfoOnException Then UpdateUserInformation() + If _ForceSaveUserInfo Then UpdateUserInformation() End Sub Friend Overridable Overloads Sub UpdateUserInformation() Implements IUserData.UpdateUserInformation UpdateUserInformation(False) @@ -1207,7 +1207,6 @@ BlockNullPicture: Protected UseClientTokens As Boolean = False Protected _ForceSaveUserData As Boolean = False Protected _ForceSaveUserInfo As Boolean = False - Protected _ForceSaveUserInfoOnException As Boolean = False Private _DownloadInProgress As Boolean = False Private _EnvirUserExists As Boolean Private _EnvirUserSuspended As Boolean @@ -1227,7 +1226,6 @@ BlockNullPicture: _DescriptionEveryTime = Settings.UpdateUserDescriptionEveryTime _ForceSaveUserData = False _ForceSaveUserInfo = False - _ForceSaveUserInfoOnException = False _EnvirUserExists = UserExists _EnvirUserSuspended = UserSuspended _EnvirCreatedByChannel = CreatedByChannel @@ -1244,8 +1242,8 @@ BlockNullPicture: Select Case Caller Case NameOf(UserExists) : If Not _EnvirUserExists = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True Case NameOf(UserSuspended) : If Not _EnvirUserSuspended = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True - Case NameOf(NameTrue) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True - Case NameOf(ID) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True + Case NameOf(NameTrue) : _EnvirChanged = True : _ForceSaveUserInfo = True + Case NameOf(ID) : _EnvirChanged = True : _ForceSaveUserInfo = True Case Else : _EnvirChanged = True End Select End If @@ -1843,7 +1841,8 @@ BlockNullPicture: updateDownCount(False) - v.File = DownloadContentDefault_ConvertWebp(ChangeFileNameByProvider(f, v), postProcessWebp) + v.File = ChangeFileNameByProvider(f, v) + v.File = DownloadContentDefault_ConvertWebp(v, postProcessWebp) v.State = UStates.Downloaded DownloadContentDefault_PostProcessing(v, f, Token) If UseMD5Comparison And (v.Type = UTypes.GIF Or v.Type = UTypes.Picture) Then @@ -1943,7 +1942,11 @@ stxt: End Function Protected Overridable Sub DownloadContentDefault_PostProcessing(ByRef m As UserMedia, ByVal File As SFile, ByVal Token As CancellationToken) End Sub - Protected Overridable Function DownloadContentDefault_ConvertWebp(ByVal WebpFile As SFile, ByVal Process As Boolean) As SFile + Protected Overridable Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile + Return DownloadContentDefault_ConvertWebp_Impl(m, Process) + End Function + Protected Overridable Function DownloadContentDefault_ConvertWebp_Impl(ByVal m As UserMedia, ByVal Process As Boolean) As SFile + Dim WebpFile As SFile = m.File Dim f As SFile = WebpFile If Process AndAlso f.Exists Then f.Path = $"{f.PathWithSeparator}Sources" @@ -1959,6 +1962,24 @@ stxt: End If Return f End Function + Protected Function DownloadContentDefault_ConvertWebp_TestImg(ByVal m As UserMedia, ByVal Process As Boolean) As SFile + If m.Type = UTypes.Picture And Settings.DownloadNativeImageFormat Then + Using testImg As New UserImage.ImageRenderer2(m.File, EDP.ReturnValue) + If testImg.IsWebP Then + Dim f As SFile = m.File + If f.Extension.IsEmptyString OrElse Not f.Extension = UserImage.ExtWebp Then + f.Extension = UserImage.ExtWebp + f = SFile.Rename(m.File, f,, EDP.ReturnValue).IfNullOrEmpty(m.File) + End If + m.File = f + Return DownloadContentDefault_ConvertWebp_Impl(m, True).IfNullOrEmpty(f) + End If + End Using + ElseIf Process Then + Return DownloadContentDefault_ConvertWebp_Impl(m, Process) + End If + Return m.File + End Function Protected Overridable Function DownloadContentDefault_ProcessDownloadException() As Boolean Return True End Function diff --git a/SCrawler/API/Bluesky/EditorExchangeOptions.vb b/SCrawler/API/Bluesky/EditorExchangeOptions.vb new file mode 100644 index 0000000..210d413 --- /dev/null +++ b/SCrawler/API/Bluesky/EditorExchangeOptions.vb @@ -0,0 +1,44 @@ +' Copyright (C) Andy https://github.com/AAndyProgram +' This program is free software: you can redistribute it and/or modify +' it under the terms of the GNU General Public License as published by +' the Free Software Foundation, either version 3 of the License, or +' (at your option) any later version. +' +' This program is distributed in the hope that it will be useful, +' but WITHOUT ANY WARRANTY +Imports SCrawler.API.Base +Imports SCrawler.Plugin.Attributes +Namespace API.Bluesky + Friend Class EditorExchangeOptions : Inherits Base.EditorExchangeOptionsBase + Friend Overrides Property SiteKey As String = BlueskySiteKey + + Friend Overridable Property DownloadModelMedia As Boolean = False + + Friend Overridable Property DownloadModelProfile As Boolean = False + Private ReadOnly Property MySettings As Object + Friend Sub New(ByVal s As SiteSettings) + DownloadModelMedia = s.DownloadModelMedia.Value + DownloadModelProfile = s.DownloadModelProfile.Value + MySettings = s + End Sub + Friend Sub New(ByVal u As UserData) + DownloadModelMedia = u.DownloadModelMedia + DownloadModelProfile = u.DownloadModelProfile + MySettings = u.HOST.Source + End Sub + Friend Overrides Sub Apply(ByRef u As UserDataBase) + MyBase.Apply(u) + If Not DownloadModelMedia And Not DownloadModelProfile Then + DownloadModelMedia = True + ElseIf DownloadModelMedia Then + DownloadModelProfile = False + Else + DownloadModelMedia = False + End If + With DirectCast(u, UserData) + .DownloadModelMedia = DownloadModelMedia + .DownloadModelProfile = DownloadModelProfile + End With + End Sub + End Class +End Namespace \ No newline at end of file diff --git a/SCrawler/API/Bluesky/SiteSettings.vb b/SCrawler/API/Bluesky/SiteSettings.vb index 2e01ec4..0ce2836 100644 --- a/SCrawler/API/Bluesky/SiteSettings.vb +++ b/SCrawler/API/Bluesky/SiteSettings.vb @@ -26,6 +26,10 @@ Namespace API.Bluesky Friend ReadOnly Property TokenUpdateTime As PropertyValue Friend ReadOnly Property TokenRefreshInterval As PropertyValue + + Friend ReadOnly Property DownloadModelMedia As PropertyValue + + Friend ReadOnly Property DownloadModelProfile As PropertyValue Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean) MyBase.New("Bluesky", "bsky.app", AccName, Temp, My.Resources.SiteResources.BlueskyIcon_32, My.Resources.SiteResources.BlueskyPic_32) @@ -38,11 +42,14 @@ Namespace API.Bluesky TokenUpdateTime = New PropertyValue(Now.AddYears(-1)) TokenRefreshInterval = New PropertyValue(120) + DownloadModelMedia = New PropertyValue(True) + DownloadModelProfile = New PropertyValue(False) + _AllowUserAgentUpdate = False UrlPatternUser = "https://bsky.app/profile/{0}" ImageVideoContains = "bsky.app" UserRegex = RParams.DMS("bsky.app/profile/([^/\?]+)", 1, EDP.ReturnValue) - UserOptionsType = GetType(EditorExchangeOptionsBase) + UserOptionsType = GetType(EditorExchangeOptions) End Sub Protected Overrides Function UserOptionsValid(ByVal Options As Object) As Boolean Return DirectCast(Options, EditorExchangeOptionsBase).SiteKey = BlueskySiteKey @@ -96,5 +103,10 @@ Namespace API.Bluesky _TokenUpdating = False End Try End Function + Friend Overrides Function IsMyUser(ByVal UserURL As String) As ExchangeOptions + Dim e As ExchangeOptions = MyBase.IsMyUser(UserURL) + If Not e.UserName.IsEmptyString AndAlso e.UserName.StartsWith("did:") Then e.UserName = e.UserName.Replace(":", "@") + Return e + End Function End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/Bluesky/UserData.vb b/SCrawler/API/Bluesky/UserData.vb index 5a429b2..7dd0d61 100644 --- a/SCrawler/API/Bluesky/UserData.vb +++ b/SCrawler/API/Bluesky/UserData.vb @@ -16,6 +16,11 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types Imports UStates = SCrawler.API.Base.UserMedia.States Namespace API.Bluesky Friend Class UserData : Inherits UserDataBase +#Region "XML names" + Private Const Name_DownloadModelMedia As String = "DownloadModelMedia" + Private Const Name_DownloadModelProfile As String = "DownloadModelProfile" + Private Const Name_ForceParseProfileInfo As String = "ForceParseProfileInfo" +#End Region #Region "Declarations" Private ReadOnly Property MySettings As SiteSettings Get @@ -28,16 +33,45 @@ Namespace API.Bluesky End Get End Property Private ReadOnly _TmpPosts2 As List(Of String) + Friend Property DownloadModelMedia As Boolean = True + Friend Property DownloadModelProfile As Boolean = False + Private Property ForceParseProfileInfo As Boolean = False #End Region #Region "Loader" + Private Sub UpdateUserOptions() + If ID.IsEmptyString AndAlso Not Name.IsEmptyString AndAlso Name.StartsWith("did@") Then + NameTrue = Name.Replace("@", ":") + ID = NameTrue + ForceParseProfileInfo = True + End If + End Sub Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean) + With Container + If Loading Then + If .Contains(Name_DownloadModelMedia) Then + DownloadModelMedia = .Value(Name_DownloadModelMedia).FromXML(Of Boolean)(True) + DownloadModelProfile = .Value(Name_DownloadModelProfile).FromXML(Of Boolean)(False) + Else + DownloadModelMedia = ParseUserMediaOnly + DownloadModelProfile = Not ParseUserMediaOnly + End If + ForceParseProfileInfo = .Value(Name_ForceParseProfileInfo).FromXML(Of Boolean)(False) + Else + If ID.IsEmptyString Then + UpdateUserOptions() + .Value(Name_UserID) = ID + End If + .Add(Name_DownloadModelMedia, DownloadModelMedia.BoolToInteger) + .Add(Name_DownloadModelProfile, DownloadModelProfile.BoolToInteger) + .Add(Name_ForceParseProfileInfo, ForceParseProfileInfo.BoolToInteger) + End If + End With End Sub Friend Overrides Function ExchangeOptionsGet() As Object - Return New EditorExchangeOptionsBase(Me) With {.SiteKey = BlueskySiteKey} + Return New EditorExchangeOptions(Me) End Function Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object) - If Not Obj Is Nothing AndAlso TypeOf Obj Is EditorExchangeOptionsBase AndAlso - DirectCast(Obj, EditorExchangeOptionsBase).SiteKey = BlueskySiteKey Then DirectCast(Obj, EditorExchangeOptionsBase).ApplyBase(Me) + If Not Obj Is Nothing AndAlso TypeOf Obj Is EditorExchangeOptions Then DirectCast(Obj, EditorExchangeOptions).Apply(Me) End Sub #End Region #Region "Initializer" @@ -66,6 +100,13 @@ Namespace API.Bluesky Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) _TmpPosts2.Clear() Try + If Not DownloadModelMedia And Not DownloadModelProfile Then + DownloadModelMedia = True + ElseIf DownloadModelMedia Then + DownloadModelProfile = False + Else + DownloadModelMedia = False + End If If Not CBool(MySettings.CookiesEnabled.Value) Then Responser.Cookies.Clear() UpdateToken(, True) _TokenUpdateCount = 0 @@ -79,7 +120,7 @@ Namespace API.Bluesky Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken) Dim URL$ = String.Empty Try - If Not IsSavedPosts And ID.IsEmptyString Then GetProfileInfo(Token) + If (Not IsSavedPosts And ID.IsEmptyString) Or ForceParseProfileInfo Then GetProfileInfo(Token) If Not IsSavedPosts And ID.IsEmptyString Then Throw New ArgumentNullException("ID", "ID is null") If UpdateToken() Then Dim nextCursor$ = String.Empty @@ -91,7 +132,9 @@ Namespace API.Bluesky n = {"bookmarks"} p = {"item"} Else - URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter=posts_and_author_threads&includePins=false&limit=99" + 'posts_and_author_threads + 'posts_with_media + URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter={IIf(DownloadModelMedia, "posts_with_media", "posts_and_author_threads")}&includePins=false&limit=99" If Not Cursor.IsEmptyString Then URL &= $"&cursor={SymbolsConverter.ASCII.EncodeSymbolsOnly(Cursor)}" n = {"feed"} p = {"post"} @@ -106,7 +149,7 @@ Namespace API.Bluesky If .ListExists Then For Each post As EContainer In .Self With post(p) - c = DefaultParser(.Self,, nextCursor) + c = DefaultParser(.Self) ',, nextCursor) Select Case c Case CInt(DateResult.Skip) * -1 : Continue For Case CInt(DateResult.Exit) * -1 : Exit Sub @@ -238,6 +281,7 @@ Namespace API.Bluesky #Region "GetProfileInfo" Private Sub GetProfileInfo(ByVal Token As CancellationToken) Try + If ForceParseProfileInfo Then ForceParseProfileInfo = False : _ForceSaveUserInfo = True If UpdateToken() Then Dim r$ = Responser.GetResponse($"https://bsky.social/xrpc/app.bsky.actor.getProfile?actor={ID.IfNullOrEmpty(NameTrue)}") TokenUpdateCountReset() @@ -344,6 +388,9 @@ Namespace API.Bluesky Protected Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile Return M3U8.Download(URL, DestinationFile, Token, Progress, Not IsSingleObjectDownload) End Function + Protected Overrides Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile + Return DownloadContentDefault_ConvertWebp_TestImg(m, Process) + End Function #End Region #Region "DownloadSingleObject" Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken) diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 291eaa5..6ddee8e 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -702,7 +702,7 @@ Namespace API.Instagram If Not IsSavedPosts Then If _UseGQL And Cursor.IsEmptyString And Not Section = Sections.SavedPosts Then UpdateTokens(True) If ID.IsEmptyString Or __idIsEmpty Or Not IsVerifiedProfile_Checked Then GetUserData(Token) - If ID.IsEmptyString Then UserExists = False : _ForceSaveUserInfoOnException = True : Throw New Plugin.ExitException("can't get user ID") + If ID.IsEmptyString Then UserExists = False : _ForceSaveUserInfo = True : Throw New Plugin.ExitException("can't get user ID") If ForceUpdateUserName Then GetUserNameById() If ForceUpdateUserInfo Then GetUserData(Token) End If diff --git a/SCrawler/API/PornHub/M3U8.vb b/SCrawler/API/PornHub/M3U8.vb index 348b996..93910c5 100644 --- a/SCrawler/API/PornHub/M3U8.vb +++ b/SCrawler/API/PornHub/M3U8.vb @@ -22,7 +22,7 @@ Namespace API.PornHub If Not r.IsEmptyString Then Dim files As List(Of Sizes) = RegexFields(Of Sizes)(r, {Regex_M3U8_FilesList}, {1, 2}, EDP.ReturnValue) Dim file$ - If files.ListExists Then files.RemoveAll(Function(f) f.Value = 0 Or (Not DownloadUHD And f.Value > 1080)) + If files.ListExists Then files.RemoveAll(Function(f) f.Value = 0 Or (Not DownloadUHD And f.Value > 1080) Or (f.Data.IsEmptyString OrElse f.Data.Contains(""""))) If files.ListExists Then files.Sort() file = files(0).Data diff --git a/SCrawler/API/PornHub/UserData.vb b/SCrawler/API/PornHub/UserData.vb index d7f4439..ccd76b4 100644 --- a/SCrawler/API/PornHub/UserData.vb +++ b/SCrawler/API/PornHub/UserData.vb @@ -258,7 +258,6 @@ Namespace API.PornHub Private _PageVideosRepeat As Integer = 0 Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) Try - UpdateM3U8URLS = False PlaylistToken = String.Empty Responser.ResetStatus() _PageVideosRepeat = 0 @@ -793,31 +792,29 @@ Namespace API.PornHub End Sub #End Region #Region "Download content" - Private UpdateM3U8URLS As Boolean = False - Private UpdateM3U8URLS_Error As Boolean = False Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken) - Try : DownloadContentDefault(Token) : Finally : UpdateM3U8URLS = False : End Try + DownloadContentDefault(Token) End Sub Protected Overloads Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile - UpdateM3U8URLS_Error = False - Return DownloadM3U8(URL, Media, DestinationFile, Token, UpdateM3U8URLS) + Return DownloadM3U8(URL, Media, DestinationFile, Token, 0) End Function Private Overloads Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, - ByVal Token As CancellationToken, ByVal Second As Boolean) As SFile + ByVal Token As CancellationToken, ByVal Round As Integer) As SFile + Const MaxRound% = 2 Try - If Second Then + If Round > 0 Then Dim r$ = Responser.Curl(Media.URL_BASE,, EDP.ReturnValue) If Not r.IsEmptyString Then Media.URL = CreateVideoURL(r).IfNullOrEmpty(URL) : URL = Media.URL End If Dim f As SFile = M3U8.Download(URL, Responser, DestinationFile, DownloadUHD, Token, Progress, Not IsSingleObjectDownload) - If Not f.Exists And Not Second Then UpdateM3U8URLS = True : f = DownloadM3U8(URL, Media, DestinationFile, Token, True) + If Not f.Exists Then f = Nothing + If f.IsEmptyString And Round < MaxRound Then f = DownloadM3U8(URL, Media, DestinationFile, Token, Round + 1) Return f Catch ex As Exception - If Not UpdateM3U8URLS_Error Then - UpdateM3U8URLS_Error = True + If Round < MaxRound Then Thread.Sleep(1000) - Return DownloadM3U8(URL, Media, DestinationFile, Token, True) + Return DownloadM3U8(URL, Media, DestinationFile, Token, Round + 1) End If Return Nothing End Try @@ -917,7 +914,6 @@ Namespace API.PornHub #End Region #Region "DownloadSingleObject" Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken) - UpdateM3U8URLS = False _TempMediaList.Add(New UserMedia(Data.URL, UTypes.VideoPre)) ReparseVideo(Token, True, Data) End Sub diff --git a/SCrawler/API/PornHub/UserExchangeOptions.vb b/SCrawler/API/PornHub/UserExchangeOptions.vb index a2a0633..5887e75 100644 --- a/SCrawler/API/PornHub/UserExchangeOptions.vb +++ b/SCrawler/API/PornHub/UserExchangeOptions.vb @@ -44,7 +44,7 @@ Namespace API.PornHub DownloadGifs = Not v = CheckState.Unchecked MySettings = s End Sub - Friend Overrides Sub Apply(ByRef u As IPSite) + Friend Overrides Sub Apply(ByRef u As UserDataBase) MyBase.Apply(u) With DirectCast(u, UserData) .DownloadUHD = DownloadUHD diff --git a/SCrawler/API/ThisVid/UserExchangeOptions.vb b/SCrawler/API/ThisVid/UserExchangeOptions.vb index d995726..3029ae4 100644 --- a/SCrawler/API/ThisVid/UserExchangeOptions.vb +++ b/SCrawler/API/ThisVid/UserExchangeOptions.vb @@ -35,7 +35,7 @@ Namespace API.ThisVid DifferentFolders = u.DifferentFolders MySettings = u.HOST.Source End Sub - Friend Overrides Sub Apply(ByRef u As IPSite) + Friend Overrides Sub Apply(ByRef u As UserDataBase) MyBase.Apply(u) With DirectCast(u, UserData) .DownloadPublic = DownloadPublic diff --git a/SCrawler/API/ThreadsNet/UserData.vb b/SCrawler/API/ThreadsNet/UserData.vb index e605c5b..ce9feed 100644 --- a/SCrawler/API/ThreadsNet/UserData.vb +++ b/SCrawler/API/ThreadsNet/UserData.vb @@ -184,7 +184,6 @@ Namespace API.ThreadsNet If uex.UserNotFound Then UserExists = False _ForceSaveUserInfo = True - _ForceSaveUserInfoOnException = True ElseIf ThrowEx Then Throw New ExitException(uex.ErrMessage) With {.SimpleLogLine = True} Else @@ -404,7 +403,6 @@ Namespace API.ThreadsNet If _IdChanged Then If Not idStr.IsEmptyString Then UserDescriptionUpdate(idStr, True, True, True) _ForceSaveUserInfo = True - _ForceSaveUserInfoOnException = True End If End If Return Valid diff --git a/SCrawler/API/TikTok/UserData.vb b/SCrawler/API/TikTok/UserData.vb index e1d6b25..00922e1 100644 --- a/SCrawler/API/TikTok/UserData.vb +++ b/SCrawler/API/TikTok/UserData.vb @@ -192,10 +192,14 @@ Namespace API.TikTok UserCache.DisposeIfReady(False) UserCache = Nothing End Sub + Private StoryStatus As Boolean = True + Private StoryStatusFound As Boolean = False Protected Overloads Overrides Sub DownloadDataF(ByVal Token As CancellationToken) ValidateCache() + StoryStatus = True + StoryStatusFound = False If GetTimeline Then DownloadDataF(Sections.Timeline, Token) - If GetStoriesUser Then DownloadDataF(Sections.UserStories, Token) + If GetStoriesUser And StoryStatus Then DownloadDataF(Sections.UserStories, Token) If GetReposts Then DownloadDataF(Sections.Reposts, Token) End Sub Protected Overloads Sub DownloadDataF(ByVal Section As Sections, ByVal Token As CancellationToken) @@ -271,8 +275,8 @@ Namespace API.TikTok Else .TempPostsList = New List(Of String) End If - .Execute(CreateGDLCommand(URL, gdlCmd)) - If Not PhotosDownloaded Then _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True + .Execute(CreateGDLCommand(URL, gdlCmd,,,, CObj(IIf(Section = Sections.Timeline, dateAfter, Nothing)))) + If Not PhotosDownloaded Then _ForceSaveUserInfo = True PhotosDownloaded = True End With End Using @@ -436,6 +440,10 @@ Namespace API.TikTok End If End With Else + If Section = Sections.Timeline And Not StoryStatusFound And j.ContainsF({0, "webapp.user-detail", "userInfo", "user"}, "UserStoryStatus") Then + StoryStatusFound = True + StoryStatus = j.ItemF({0, "webapp.user-detail", "userInfo", "user"}, "UserStoryStatus").Value.FromXML(Of Boolean)(False) + End If With j.ItemF({0, "webapp.video-detail", "itemInfo", "itemStruct"}) If .ListExists Then postID = .Value("id") @@ -604,13 +612,17 @@ Namespace API.TikTok #End Region #Region "GDL Support" Private Function CreateGDLCommand(ByVal URL As String, Optional ByVal SectionCommand As String = Nothing, - Optional ByVal IsDownload As Boolean = False, Optional ByVal Output As SFile = Nothing) As String + Optional ByVal IsDownload As Boolean = False, Optional ByVal Output As SFile = Nothing, + Optional ByVal DateBefore As Date? = Nothing, Optional ByVal DateAfter As Date? = Nothing) As String Dim command$ = $"""{Settings.GalleryDLFile}"" " If Not IsDownload Then command &= "--verbose --no-download --no-skip --write-pages " Else command &= $"--dest ""{Output.PathNoSeparator}"" " End If + If DateBefore.HasValue Or DateAfter.HasValue Then + With If(DateAfter, DateBefore).Value : command &= $"--filter ""date {IIf(DateAfter.HasValue, ">", "<")} datetime({ .Year}, { .Month}, { .Day}) or terminate()"" " : End With + End If If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then command &= "--no-mtime " command &= $"{SectionCommand} {URL}" Return command diff --git a/SCrawler/API/Twitter/UserData.vb b/SCrawler/API/Twitter/UserData.vb index c7ddcf4..df9a7de 100644 --- a/SCrawler/API/Twitter/UserData.vb +++ b/SCrawler/API/Twitter/UserData.vb @@ -523,7 +523,7 @@ Namespace API.Twitter With .ItemF({0, "content", "items", 0, "item", "itemContent", "tweet_results", "result", "tweet", "community_results", "result"}) If .ListExists Then If ID = .Value("id_str") Then - UserSiteNameUpdate(.Value("name")) + UserSiteNameUpdate(.Value("name"), True) UserDescriptionUpdate(.Value("description")) icon = .Value({"custom_banner_media", "media_info"}, "original_img_url"). @@ -543,12 +543,12 @@ Namespace API.Twitter If .ListExists Then If ID.IsEmptyString Then ID = .Value("rest_id") icon = .Value({"avatar"}, "image_url") - UserSiteNameUpdate(.Value({"core"}, "name")) + UserSiteNameUpdate(.Value({"core"}, "name"), True) Dim tScreenName$ = .Value({"core"}, "screen_name") With .Item({"legacy"}) If .ListExists Then If onlyUpdateUser Then - If Not NameTrue = tScreenName Or 1 = 1 Then + If Not NameTrue = tScreenName Then Dim uStr$ = $"username changed from '{NameTrue}' to '{tScreenName}'" LogError(Nothing, uStr) UserDescriptionUpdate(uStr, True, True, True) @@ -556,7 +556,7 @@ Namespace API.Twitter NameTrue = tScreenName End If If .Value("screen_name").IfNullOrEmpty(tScreenName).StringToLower = NameTrue.ToLower Then - UserSiteNameUpdate(.Value("name")) + UserSiteNameUpdate(.Value("name"), True) UserDescriptionUpdate(.Value("description")) If icon.IsEmptyString Then icon = .Value("profile_image_url_https") @@ -1190,10 +1190,12 @@ nextpIndx: Dim cache As CacheKeeper = Nothing Try If ContentMissingExists Or (_ReparseLikes And LikesPosts.Count > 0) Then + Const __entries$ = "entries" Dim m As UserMedia, mTmp As UserMedia Dim PostDate$ Dim nodes As List(Of String()) = GetContainerSubnodes() Dim node$() + Dim entriesNode As Predicate(Of EContainer) = Function(ee) ee.Contains(__entries) Dim j As EContainer, n As EContainer Dim f As SFile Dim i%, ii% @@ -1229,7 +1231,7 @@ nextpIndx: f = GDLRenameFile(files(ii), ii) j = JsonDocument.Parse(f.GetText) If Not j Is Nothing Then - With j.ItemF({"data", 0, "instructions", 0, "entries"}) + With j.ItemF({"data", 0, "instructions", entriesNode, __entries}) If .ListExists Then If IsSingleObjectDownload Or DownloadBroadcasts Then mTmp = ExtractBroadcast(.Self, m.Post.ID, String.Empty, nodes) diff --git a/SCrawler/API/Xhamster/UserData.vb b/SCrawler/API/Xhamster/UserData.vb index ba5be9c..2a5ad79 100644 --- a/SCrawler/API/Xhamster/UserData.vb +++ b/SCrawler/API/Xhamster/UserData.vb @@ -298,6 +298,7 @@ Namespace API.Xhamster containerNodes.Add({"videoListComponent", "videoThumbProps"}) Else containerNodes.Add({"userGalleriesCollection"}) + containerNodes.Add({"contentComponent", "items"}) End If End If @@ -799,6 +800,9 @@ Namespace API.Xhamster Using ytdlp As New YTDLP.YTDLPBatch(TokenPersonal,, DestinationFile) : ytdlp.Encoding = Settings.CMDEncoding : ytdlp.Execute(cmd) : End Using Return DestinationFile End Function + Protected Overrides Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile + Return DownloadContentDefault_ConvertWebp_TestImg(m, Process) + End Function #End Region #Region "Create media" Private Function ExtractMedia(ByVal j As EContainer, ByVal t As UTypes, Optional ByVal UrlNode As String = "pageURL", diff --git a/SCrawler/API/Xhamster/UserExchangeOptions.vb b/SCrawler/API/Xhamster/UserExchangeOptions.vb index fbfd5f7..9c58aeb 100644 --- a/SCrawler/API/Xhamster/UserExchangeOptions.vb +++ b/SCrawler/API/Xhamster/UserExchangeOptions.vb @@ -23,7 +23,7 @@ Namespace API.Xhamster MyBase.New(s) GetMoments = s.GetMoments.Value End Sub - Friend Overrides Sub Apply(ByRef u As IPSite) + Friend Overrides Sub Apply(ByRef u As UserDataBase) MyBase.Apply(u) DirectCast(u, UserData).GetMoments = GetMoments End Sub diff --git a/SCrawler/Download/Feed/FeedMedia.vb b/SCrawler/Download/Feed/FeedMedia.vb index 94f9974..93044a1 100644 --- a/SCrawler/Download/Feed/FeedMedia.vb +++ b/SCrawler/Download/Feed/FeedMedia.vb @@ -12,6 +12,7 @@ Imports PersonalUtilities.Forms Imports PersonalUtilities.Tools Imports SCrawler.API.Base Imports UserMediaD = SCrawler.DownloadObjects.TDownloader.UserMediaD +Imports ImageRenderer2 = SCrawler.UserImage.ImageRenderer2 Namespace DownloadObjects Public Class FeedMedia @@ -166,42 +167,6 @@ Namespace DownloadObjects Public Sub New() InitializeComponent() End Sub - Private Class ImageRenderer2 : Inherits ImageRenderer - Friend NativeFormat As String = Nothing - Friend ImgErr As Exception = Nothing - Friend Sub New(ByVal ImgPath As SFile, Optional ByVal e As ErrorsDescriber = Nothing) - MyBase.New() - Try - If ImgPath.Exists(SFO.File, False) Then - OriginalImageBytes = SFile.GetBytes(ImgPath, EDP.ThrowException) - Try - OriginalImage = GetImage(OriginalImageBytes) - Catch exInternal As Exception - HasError = True - ImgErr = exInternal - NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue) - End Try - End If - Address = ImgPath - Catch ex As Exception - HasError = True - NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue) - If Not e.Exists Then e = EDP.ThrowException - ErrorsDescriber.Execute(e, ex, $"ImageRenderer2.New({ImgPath})") - End Try - End Sub - Friend Shared Function GetTrueFormat(ByVal Img() As Byte, Optional ByVal e As ErrorsDescriber = Nothing) As String - Try - Using ms As New MemoryStream(Img, 0, Img.Length) - Return System.Windows.Media.Imaging.BitmapDecoder.Create(ms, Windows.Media.Imaging.BitmapCreateOptions.PreservePixelFormat, - Windows.Media.Imaging.BitmapCacheOption.OnLoad).Metadata.Format - End Using - Catch ex As Exception - If Not e.Exists Then e = EDP.ThrowException - Return ErrorsDescriber.Execute(e, ex, "[ImageRenderer2.GetTrueFormat()]") - End Try - End Function - End Class Friend Sub New(ByVal Media As UserMediaD, ByVal Width As Integer, ByVal Height As Integer, ByVal IsSession As Boolean, ByVal ExtractText As Boolean) Try InitializeComponent() diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index a27b913..4a7c0b4 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/SCrawler.vbproj b/SCrawler/SCrawler.vbproj index 93a1b54..ade2942 100644 --- a/SCrawler/SCrawler.vbproj +++ b/SCrawler/SCrawler.vbproj @@ -194,6 +194,7 @@ + diff --git a/SCrawler/UserImage.vb b/SCrawler/UserImage.vb index 48522c8..b91f8fa 100644 --- a/SCrawler/UserImage.vb +++ b/SCrawler/UserImage.vb @@ -150,4 +150,45 @@ Friend Class UserImage : Inherits ImageRenderer Private Shared Function ConvertWebpTryImageMagick(ByVal InitFile As SFile, ByVal DestFile As SFile) As Boolean Return ImageRendererExt.ConvertWebp(InitFile, DestFile, EDP.SendToLog + EDP.ReturnValue) End Function + Friend Class ImageRenderer2 : Inherits ImageRenderer + Friend NativeFormat As String = Nothing + Friend ImgErr As Exception = Nothing + Friend ReadOnly Property IsWebP As Boolean + Get + Return NativeFormat.IfNullOrEmpty(ExtJpg).ToLower = ExtWebp + End Get + End Property + Friend Sub New(ByVal ImgPath As SFile, Optional ByVal e As ErrorsDescriber = Nothing) + MyBase.New() + Try + If ImgPath.Exists(SFO.File, False) Then + OriginalImageBytes = SFile.GetBytes(ImgPath, EDP.ThrowException) + Try + OriginalImage = GetImage(OriginalImageBytes) + Catch exInternal As Exception + HasError = True + ImgErr = exInternal + NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue) + End Try + End If + Address = ImgPath + Catch ex As Exception + HasError = True + NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue) + If Not e.Exists Then e = EDP.ThrowException + ErrorsDescriber.Execute(e, ex, $"ImageRenderer2.New({ImgPath})") + End Try + End Sub + Friend Shared Function GetTrueFormat(ByVal Img() As Byte, Optional ByVal e As ErrorsDescriber = Nothing) As String + Try + Using ms As New MemoryStream(Img, 0, Img.Length) + Return System.Windows.Media.Imaging.BitmapDecoder.Create(ms, Windows.Media.Imaging.BitmapCreateOptions.PreservePixelFormat, + Windows.Media.Imaging.BitmapCacheOption.OnLoad).Metadata.Format + End Using + Catch ex As Exception + If Not e.Exists Then e = EDP.ThrowException + Return ErrorsDescriber.Execute(e, ex, "[ImageRenderer2.GetTrueFormat()]") + End Try + End Function + End Class End Class \ No newline at end of file