From 9ba78d2fa04a5afec495134dce0c90f14f6a3d78 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Wed, 13 May 2026 20:45:18 +0300 Subject: [PATCH] 2026.5.13.0 API.Base: add 'IThumbList' interface; add the 'ReparseMissing_ClearList' function to the base class API.Instagram: protect all necessary controls; fix error getting user info API.OnlyFans: add the 'CookieDateProvider' class to avoid cookies error API.PornHub: add image and title additional regex; fix subscription downloading bug API.ThisVid: add 'ReparseMissing' function Feed: add the most frequently used buttons to the top FeedMedia: fix subscription bug --- Changelog.md | 20 ++++- SCrawler/API/Base/Declarations.vb | 3 + SCrawler/API/Base/UserDataBase.vb | 6 ++ SCrawler/API/Instagram/Declarations.vb | 1 + SCrawler/API/Instagram/SiteSettings.vb | 14 ++-- SCrawler/API/Instagram/UserData.GQL.vb | 31 +++++-- SCrawler/API/Instagram/UserData.vb | 3 +- SCrawler/API/OnlyFans/Declarations.vb | 18 +++++ SCrawler/API/OnlyFans/SiteSettings.vb | 1 + SCrawler/API/PornHub/Declarations.vb | 5 ++ SCrawler/API/PornHub/UserData.vb | 92 ++++++++++++++------- SCrawler/API/ThisVid/UserData.vb | 29 ++++++- SCrawler/Download/Feed/DownloadFeedForm.vb | 94 ++++++++++++++++++---- SCrawler/Download/Feed/FeedMedia.vb | 16 +++- SCrawler/My Project/AssemblyInfo.vb | 4 +- 15 files changed, 273 insertions(+), 64 deletions(-) diff --git a/Changelog.md b/Changelog.md index 514cc93..56d6ffd 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,13 +2,31 @@ - [ffmpeg](https://github.com/AAndyProgram/SCrawler/wiki/Settings#ffmpeg) - x64 version - [release](https://github.com/GyanD/codexffmpeg/releases/tag/5.1.2); [zip](https://github.com/GyanD/codexffmpeg/releases/download/5.1.2/ffmpeg-5.1.2-full_build.zip); **version `5.1.2-full_build-www.gyan.dev`** - x86 version - [release](https://github.com/yt-dlp/FFmpeg-Builds/releases/tag/autobuild-2022-11-30-12-57); [zip](https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-2022-11-30-12-57/ffmpeg-N-109274-gd7a5f068c2-win32-gpl.zip); **version `N-109457-geeb280f351-20221226`** -- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.9** +- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.32.0** - [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.03.17** - [Deno](https://github.com/AAndyProgram/SCrawler/wiki/Settings#deno) - latest *(`2.0.0` or higher)* - [OF-Scraper](https://github.com/AAndyProgram/SCrawler/wiki/Settings#of-scraper) - **3.12.9** ([release](https://github.com/datawhores/OF-Scraper/releases/tag/3.12.9)) # 2026 +## 2026.5.13.0 + +*2026-05-13* + +- Added + - Sites + - ThisVid: download undownloaded videos + - Feed: the most frequently used buttons are added to the top + - Minor improvements +- Updated + - gallery-dl up to version **1.32.0** +- Fixed + - Sites: + - **Instagram: data is not downloading** + - OnlyFans: cookies error + - PornHub: subscriptions are not downloading + - Minor bugs + ## 2026.3.23.0 *2026-03-23* diff --git a/SCrawler/API/Base/Declarations.vb b/SCrawler/API/Base/Declarations.vb index 24956b1..6d60778 100644 --- a/SCrawler/API/Base/Declarations.vb +++ b/SCrawler/API/Base/Declarations.vb @@ -11,6 +11,9 @@ Imports PersonalUtilities.Forms Imports PersonalUtilities.Functions.RegularExpressions Namespace API.Base Friend Module Declarations + Friend Interface IThumbList + Property Thumbs As List(Of String) + End Interface Friend Const UserLabelName As String = "User" Friend Const SearchRequestLabelName As String = "Search request" Friend ReadOnly LNC As New ListAddParams(LAP.NotContainsOnly) diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index b7541f5..7add9c4 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -1550,6 +1550,12 @@ BlockNullPicture: ''' Protected Overridable Sub ReparseMissing(ByVal Token As CancellationToken) End Sub + Protected Sub ReparseMissing_ClearList(ByRef rList As List(Of Integer)) + If rList.Count > 0 Then + For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(rList(i)) : Next + rList.Clear() + End If + End Sub #End Region #Region "MD5 support" Private Const VALIDATE_MD5_ERROR As String = "VALIDATE_MD5_ERROR" diff --git a/SCrawler/API/Instagram/Declarations.vb b/SCrawler/API/Instagram/Declarations.vb index ec24710..28dc502 100644 --- a/SCrawler/API/Instagram/Declarations.vb +++ b/SCrawler/API/Instagram/Declarations.vb @@ -20,6 +20,7 @@ Namespace API.Instagram Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]" Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue) Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue) + Friend ReadOnly Regex_BlockVersionID As RParams = RParams.DMS("""versioningID"":""([^""]+)""", 1, EDP.ReturnValue) Friend ReadOnly Regex_ProfileID As RParams = RParams.DMS("profilePage_(\d+)", 1, EDP.ReturnValue) Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean) Const r_wwwClaimName$ = "x-ig-set-www-claim" diff --git a/SCrawler/API/Instagram/SiteSettings.vb b/SCrawler/API/Instagram/SiteSettings.vb index e3a32e8..73690eb 100644 --- a/SCrawler/API/Instagram/SiteSettings.vb +++ b/SCrawler/API/Instagram/SiteSettings.vb @@ -70,7 +70,7 @@ Namespace API.Instagram Friend Const Header_Browser As String = "Sec-Ch-Ua" Friend Const Header_BrowserExt As String = "Sec-Ch-Ua-Full-Version-List" Friend Const Header_Platform_Verion As String = "Sec-Ch-Ua-Platform-Version" - + Friend ReadOnly Property HH_CSRF_TOKEN As PropertyValue Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String @@ -78,7 +78,7 @@ Namespace API.Instagram End Function Friend ReadOnly Property HH_IG_APP_ID As PropertyValue - + Friend ReadOnly Property HH_ASBD_ID As PropertyValue 'PropertyOption(ControlText:="x-ig-www-claim", IsAuth:=True, AllowNull:=True) @@ -89,16 +89,16 @@ Namespace API.Instagram Return Not v.IsEmptyString AndAlso v = "0" End Get End Property - Private ReadOnly Property HH_BROWSER As PropertyValue - Private ReadOnly Property HH_BROWSER_EXT As PropertyValue - Private ReadOnly Property HH_PLATFORM As PropertyValue - Private ReadOnly Property HH_USER_AGENT As PropertyValue Friend Overrides Function BaseAuthExists() As Boolean @@ -146,7 +146,7 @@ Namespace API.Instagram Private ReadOnly Property TokenUpdateIntervalProvider As IFormatProvider #End Region - + Friend ReadOnly Property USE_GQL As PropertyValue Friend ReadOnly Property USE_GQL_UserData As PropertyValue diff --git a/SCrawler/API/Instagram/UserData.GQL.vb b/SCrawler/API/Instagram/UserData.GQL.vb index 22cde45..61c52ca 100644 --- a/SCrawler/API/Instagram/UserData.GQL.vb +++ b/SCrawler/API/Instagram/UserData.GQL.vb @@ -27,13 +27,17 @@ Namespace API.Instagram Token_dtsg = String.Empty Token_lsd = String.Empty End Sub + Protected Property BlockVersionID As String = String.Empty #End Region #Region "Headers" Friend Const GQL_HEADER_FB_FRINDLY_NAME As String = "x-fb-friendly-name" Friend Const GQL_HEADER_FB_LSD As String = "x-fb-lsd" + Friend Const GQL_HEADER_BLOCK_VERSION_ID As String = "x-bloks-version-id" + Friend Const GQL_HEADER_ROOT_FIELD_NAME As String = "x-root-field-name" + Friend Const GQL_HEADER_ROOT_FIELD_NAME_Value As String = "fetch__XDTUserDict" #End Region #Region "Data constants" - Private Const GQL_UserData_DocId As String = "7381344031985950" + Private Const GQL_UserData_DocId As String = "35710877621861450" '"7381344031985950" Private Const GQL_UserData_FbFriendlyName As String = "PolarisProfilePageContentQuery" Private Const GQL_Highlights_DocId As String = "8298007123561120" @@ -61,18 +65,31 @@ Namespace API.Instagram Private Const GQL_URL_Q As String = "https://www.instagram.com/graphql/query" #End Region #Region "Download functions" - Protected Sub UpdateHeadersGQL(ByVal HeaderValue As String) - Responser.Headers.Add(GQL_HEADER_FB_FRINDLY_NAME, HeaderValue) - Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd) + Protected Sub UpdateHeadersGQL(ByVal HeaderValue As String, Optional ByVal Add As Boolean = True) + With Responser.Headers + If Add Then + .Add(GQL_HEADER_FB_FRINDLY_NAME, HeaderValue) + .Add(GQL_HEADER_FB_LSD, Token_lsd) + '.Add(GQL_HEADER_BLOCK_VERSION_ID, BlockVersionID) + '.Add(GQL_HEADER_ROOT_FIELD_NAME, GQL_HEADER_ROOT_FIELD_NAME_Value) + Else + .Remove(GQL_HEADER_FB_FRINDLY_NAME) + .Remove(GQL_HEADER_FB_LSD) + '.Remove(GQL_HEADER_BLOCK_VERSION_ID) + '.Remove(GQL_HEADER_ROOT_FIELD_NAME) + End If + End With End Sub ' Private Function GetUserDataGQL(ByVal Token As CancellationToken) As String + 'Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserData_DocId, Token_lsd, Token_dtsg_Var, GQL_UserData_FbFriendlyName, + ' SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""id"":""{ID}"",""relay_header"":false,""render_surface"":""PROFILE""" & "}")) Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserData_DocId, Token_lsd, Token_dtsg_Var, GQL_UserData_FbFriendlyName, - SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""id"":""{ID}"",""relay_header"":false,""render_surface"":""PROFILE""" & "}")) + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""enable_integrity_filters"":true,""id"":""{ID}"",""__relay_internal__pv__PolarisCannesGuardianExperienceEnabledrelayprovider"":true,""__relay_internal__pv__PolarisCASB976ProfileEnabledrelayprovider"":false,""__relay_internal__pv__PolarisWebSchoolsEnabledrelayprovider"":false,""__relay_internal__pv__PolarisRepostsConsumptionEnabledrelayprovider"":true" & "}")) UpdateRequestNumber() ChangeResponserMode(True) UpdateHeadersGQL(GQL_UserData_FbFriendlyName) - Dim r$ = Responser.GetResponse(GQL_URL, vars) + Dim r$ = Responser.GetResponse(GQL_URL_Q, vars) Return r 'If Not r.IsEmptyString Then ' Using j As EContainer = JsonDocument.Parse(r) @@ -344,6 +361,7 @@ Namespace API.Instagram .MatchSub = 1 .WhatGet = RegexReturn.Value End With + BlockVersionID = RegexReplace(r, Regex_BlockVersionID) For Each tt In tokens If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then Exit For @@ -362,6 +380,7 @@ Namespace API.Instagram Case 1 Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg) Token_lsd = RegexReplace(r, Regex_UserToken_lsd) + BlockVersionID = RegexReplace(r, Regex_BlockVersionID) End Select If Not ValidateBaseTokens() And Attempt = 0 Then ParseTokens(r, Attempt + 1) End If diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 6ddee8e..a4f97eb 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -362,8 +362,7 @@ Namespace API.Instagram With .Headers .Remove("origin") .Remove("authority") - .Remove(GQL_HEADER_FB_FRINDLY_NAME) - .Remove(GQL_HEADER_FB_LSD) + UpdateHeadersGQL(String.Empty, False) Dim hv$ = MySiteSettings.Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest)).IfNullOrEmpty("empty") .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, hv)) hv = MySiteSettings.Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode)).IfNullOrEmpty("cors") diff --git a/SCrawler/API/OnlyFans/Declarations.vb b/SCrawler/API/OnlyFans/Declarations.vb index b697206..6ddb955 100644 --- a/SCrawler/API/OnlyFans/Declarations.vb +++ b/SCrawler/API/OnlyFans/Declarations.vb @@ -17,5 +17,23 @@ Namespace API.OnlyFans {{"files", "full", "url"}} } Friend Property Rules As DynamicRulesEnv + Friend Class CookieDateProvider : Implements ICustomProvider + Private ReadOnly DefaultProvider As IFormatProvider + Friend Sub New() + DefaultProvider = PersonalUtilities.Tools.Web.Cookies.CookieKeeper.DateProviderDefault + End Sub + Friend Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider, + Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object Implements ICustomProvider.Convert + If Not IsNothing(Value) AndAlso TypeOf Value Is String AndAlso Not CStr(Value).IsEmptyString Then + Dim v$ = Value + If v.Contains(",_") Then v = v.Substring(0, v.IndexOf(",_")) + Value = v + End If + Return AConvert(Of Date)(Value, DefaultProvider, Nothing) + End Function + Private Function GetFormat(ByVal FormatType As Type) As Object Implements IFormatProvider.GetFormat + Throw New NotImplementedException("'GetFormat' is not available in the 'CookieDateProvider'") + End Function + End Class End Module End Namespace \ No newline at end of file diff --git a/SCrawler/API/OnlyFans/SiteSettings.vb b/SCrawler/API/OnlyFans/SiteSettings.vb index f2f61db..1aa76cf 100644 --- a/SCrawler/API/OnlyFans/SiteSettings.vb +++ b/SCrawler/API/OnlyFans/SiteSettings.vb @@ -253,6 +253,7 @@ Namespace API.OnlyFans .CookiesUpdateMode = CookieKeeper.UpdateModes.Disabled .Cookies.ChangedAllowInternalDrop = False .Cookies.Changed = False + .Cookies.DateProvider = New CookieDateProvider With .Headers .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform)) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile)) diff --git a/SCrawler/API/PornHub/Declarations.vb b/SCrawler/API/PornHub/Declarations.vb index 7068542..8c92bdf 100644 --- a/SCrawler/API/PornHub/Declarations.vb +++ b/SCrawler/API/PornHub/Declarations.vb @@ -24,7 +24,12 @@ Namespace API.PornHub 0, RegexOptions.Singleline, RegexReturn.List, EDP.ReturnValue, UnicodeHexConverter) Friend ReadOnly RegexVideo_Video_VideoKey As RParams = RParams.DMS("viewkey=([\w\d]+)", 1, EDP.ReturnValue) Friend ReadOnly RegexVideoPageTitle As RParams = RParams.DMS("meta (property|name)=""[^:]+?:title"" content=""([^""]+)""", 2, EDP.ReturnValue) + Friend ReadOnly RegexVideoPageTitle2 As RParams = RParams.DMS("\([^\<]+)\", 1, EDP.ReturnValue) + Friend ReadOnly RegexVideoPageTitle3 As RParams = RParams.DMS("videoTitle...([^""]+)", 1, EDP.ReturnValue) + Friend ReadOnly RegexVideoPageTitle_NoTitle As RParams = RParams.DMS("(?<=(\=|/))([\w\d]+)(?=(\?|\&|\Z))", 2, EDP.ReturnValue) Friend ReadOnly RegexDataToken As RParams = RParams.DMS("data-token=""([^""]+)", 1, EDP.ReturnValue) + Friend ReadOnly RegexVideoAdditImg As RParams = RParams.DMS("twitter.image..content=""([^""]+)""", 1, EDP.ReturnValue) + Friend ReadOnly RegexVideoAdditImg2 As RParams = RParams.DMS("thumbnailUrl""[^""]*?""([^""]+)""", 1, EDP.ReturnValue) #End Region #Region "Declarations M3U8" Friend ReadOnly Regex_M3U8_FilesList As RParams = RParams.DM("RESOLUTION=\d+x(\d+).*?[\r\n]*?(.+?m3u8.*)", 0, RegexReturn.List, EDP.ReturnValue) diff --git a/SCrawler/API/PornHub/UserData.vb b/SCrawler/API/PornHub/UserData.vb index ccd76b4..b6c2eb6 100644 --- a/SCrawler/API/PornHub/UserData.vb +++ b/SCrawler/API/PornHub/UserData.vb @@ -6,13 +6,14 @@ ' ' This program is distributed in the hope that it will be useful, ' but WITHOUT ANY WARRANTY +Imports System.Security.Policy Imports System.Threading -Imports SCrawler.API.Base -Imports SCrawler.API.YouTube.Objects -Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Functions.RegularExpressions +Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Tools.Web.Clients Imports PersonalUtilities.Tools.Web.Documents.JSON +Imports SCrawler.API.Base +Imports SCrawler.API.YouTube.Objects Imports UTypes = SCrawler.API.Base.UserMedia.Types Namespace API.PornHub Friend Class UserData : Inherits UserDataBase : Implements IPSite @@ -96,6 +97,9 @@ Namespace API.PornHub Return Me End Function End Structure + Private Structure PHObj : Implements IThumbList + Friend Property Thumbs As List(Of String) Implements IThumbList.Thumbs + End Structure #End Region #Region "Enums" Private Enum VideoTypes @@ -647,6 +651,21 @@ Namespace API.PornHub #End Region #End Region #Region "ReparseVideo" + Private Function GetVideoTitle(ByVal r As String, ByVal URL As String) As String + Dim tmpName$ = String.Empty + Dim rr As RParams + For i As Byte = 0 To 3 + Select Case i + Case 0 : rr = RegexVideoPageTitle + Case 1 : rr = RegexVideoPageTitle2 + Case 2 : rr = RegexVideoPageTitle3 + Case Else : rr = RegexVideoPageTitle_NoTitle + End Select + tmpName = RegexReplace(IIf(i = 3, URL, r), rr) + If Not tmpName.IsEmptyString Then Exit For + Next + Return tmpName + End Function Protected Overloads Overrides Sub ReparseVideo(ByVal Token As CancellationToken) If IsSubscription Then ReparseVideoSubscriptions(Token) @@ -662,6 +681,7 @@ Namespace API.PornHub If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(tm) tm.Type = UTypes.VideoPre) Then Dim m As UserMedia Dim r$, NewUrl$, tmpName$ + Dim isCurl% ProgressPre.ChangeMax(_TempMediaList.Count) For i% = _TempMediaList.Count - 1 To 0 Step -1 ProgressPre.Perform() @@ -670,27 +690,30 @@ Namespace API.PornHub ThrowAny(Token) Try URL = m.URL - r = Responser.Curl(URL) - If Not r.IsEmptyString Then - NewUrl = CreateVideoURL(r) - If NewUrl.IsEmptyString Then - Throw New Exception With {.HelpLink = ERR_NEW_URL} - Else - m.URL = NewUrl - m.Type = UTypes.m3u8 - If CreateFileName Then - tmpName = RegexReplace(r, RegexVideoPageTitle) - If Not tmpName.IsEmptyString Then - If Not Data Is Nothing Then Data.Title = tmpName - m.File.Name = TitleHtmlConverter(tmpName) - m.File.Extension = "mp4" + For isCurl = 1 To 0 Step -1 + If CBool(isCurl) Then r = Responser.Curl(URL) Else r = Responser.GetResponse(URL) + If Not r.IsEmptyString Then + NewUrl = CreateVideoURL(r) + If NewUrl.IsEmptyString Then + Throw New Exception With {.HelpLink = ERR_NEW_URL} + Else + m.URL = NewUrl + m.Type = UTypes.m3u8 + If CreateFileName Then + tmpName = GetVideoTitle(r, URL) + If Not tmpName.IsEmptyString Then + If Not Data Is Nothing Then Data.Title = tmpName + m.File.Name = TitleHtmlConverter(tmpName) + m.File.Extension = "mp4" + End If End If + _TempMediaList(i) = m End If - _TempMediaList(i) = m + Exit For + ElseIf Not CBool(isCurl) Then + _TempMediaList.RemoveAt(i) End If - Else - _TempMediaList.RemoveAt(i) - End If + Next Catch mid_ex As Exception If mid_ex.HelpLink = ERR_NEW_URL OrElse DownloadingException(mid_ex, "") = 1 Then m.State = UserMedia.States.Missing @@ -712,6 +735,10 @@ Namespace API.PornHub Dim m As UserMedia Dim r$, URL$, tmpName$, thumb$ Dim c% = 0 + Dim thumbObj As PHObj + Dim thumbAdded As Boolean + Dim rp As RParams + Dim rpa() As RParams = {RegexVideoAdditImg, RegexVideoAdditImg2} Dim rErr As New ErrorsDescriber(EDP.ReturnValue) Progress.Maximum += _TempMediaList.Count For i% = _TempMediaList.Count - 1 To 0 Step -1 @@ -726,10 +753,24 @@ Namespace API.PornHub If Not r.IsEmptyString Then m.Type = UTypes.m3u8 + thumbObj = New PHObj With {.Thumbs = New List(Of String)} + thumbAdded = False thumb = RegexReplace(r, Regex_VideosThumb_OG_IMAGE) - If Not thumb.IsEmptyString Then m.URL = thumb + If Not thumb.IsEmptyString Then m.URL = thumb : thumbAdded = True + For Each rp In rpa + thumb = RegexReplace(r, rp) + If Not thumb.IsEmptyString Then + If Not thumbAdded Then + m.URL = thumb + thumbAdded = True + Else + thumbObj.Thumbs.Add(thumb) + End If + End If + Next + If thumbObj.Thumbs.Count > 0 Then m.Object = thumbObj - tmpName = RegexReplace(r, RegexVideoPageTitle) + tmpName = GetVideoTitle(r, URL) If Not tmpName.IsEmptyString Then m.File.Name = TitleHtmlConverter(tmpName) m.File.Extension = "mp4" @@ -784,10 +825,7 @@ Namespace API.PornHub Catch ex As Exception ProcessException(ex, Token, "missing data downloading error") Finally - If rList.Count > 0 Then - For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(rList(i)) : Next - rList.Clear() - End If + ReparseMissing_ClearList(rList) End Try End Sub #End Region diff --git a/SCrawler/API/ThisVid/UserData.vb b/SCrawler/API/ThisVid/UserData.vb index 4f3f761..33c7daa 100644 --- a/SCrawler/API/ThisVid/UserData.vb +++ b/SCrawler/API/ThisVid/UserData.vb @@ -403,6 +403,30 @@ Namespace API.ThisVid End Sub #End Region #Region "ReparseVideo" + Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken) + Dim rList As New List(Of Integer) + Try + If Not IsSubscription And ContentMissingExists Then + Dim m As UserMedia + ProgressPre.ChangeMax(_ContentList.Count) + For i% = 0 To _ContentList.Count - 1 + ProgressPre.Perform() + m = _ContentList(i) + If m.State = UserMedia.States.Missing AndAlso Not m.URL_BASE.IsEmptyString Then + ThrowAny(Token) + m.URL = m.URL_BASE + m.Type = UserMedia.Types.VideoPre + _TempMediaList.ListAddValue(m, LNC) + rList.Add(i) + End If + Next + End If + Catch ex As Exception + ProcessException(ex, Token, "missing data downloading error") + Finally + ReparseMissing_ClearList(rList) + End Try + End Sub Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken) If IsSubscription Then ReparseVideoSubscriptions(Token) @@ -440,11 +464,12 @@ Namespace API.ThisVid If n.IsEmptyString Then n = u.Post.ID If n.IsEmptyString Then n = "VideoFile" u.File = $"{n}.mp4" - If u.URL.IsEmptyString OrElse (Not u.Post.ID.IsEmptyString AndAlso _TempPostsList.Contains(u.Post.ID)) Then + If Not u.State = UserMedia.States.Missing AndAlso + (u.URL.IsEmptyString OrElse (Not u.Post.ID.IsEmptyString AndAlso _TempPostsList.Contains(u.Post.ID))) Then _TempMediaList.RemoveAt(i) Else u.Type = UserMedia.Types.Video - _TempPostsList.Add(u.Post.ID) + _TempPostsList.ListAddValue(u.Post.ID, LNC) _TempMediaList(i) = u End If e.Dispose() diff --git a/SCrawler/Download/Feed/DownloadFeedForm.vb b/SCrawler/Download/Feed/DownloadFeedForm.vb index fd2595d..846e3e3 100644 --- a/SCrawler/Download/Feed/DownloadFeedForm.vb +++ b/SCrawler/Download/Feed/DownloadFeedForm.vb @@ -26,7 +26,16 @@ Namespace DownloadObjects Friend WithEvents MyDefs As DefaultFormOptions Private WithEvents MyRange As RangeSwitcherToolbar(Of UserMediaD) Private ReadOnly DataList As List(Of UserMediaD) +#Region "Buttons" Private WithEvents BTT_DELETE_SELECTED As ToolStripButton + Private WithEvents BTT_COPY_SELECTED_T As ToolStripButton + Private WithEvents BTT_MOVE_SELECTED_T As ToolStripButton + Private WithEvents BTT_FAV_T As ToolStripButton + Private WithEvents BTT_FEED_ADD_T As ToolStripDropDownButton + Private WithEvents BTT_SELECT_ALL_T As ToolStripButton + Private WithEvents BTT_SELECT_NONE_T As ToolStripButton + Private WithEvents BTT_INVERT_T As ToolStripButton +#End Region Private DataRows As Integer = 10 Private DataColumns As Integer = 1 Private FeedEndless As Boolean = False @@ -157,13 +166,61 @@ Namespace DownloadObjects MyRange = New RangeSwitcherToolbar(Of UserMediaD)(ToolbarTOP) DataList = New List(Of UserMediaD) LoadedFeedNames = New List(Of String) + BTT_DELETE_SELECTED = New ToolStripButton With { - .Text = "Delete selected", + .Text = "Delete", .AutoToolTip = True, - .ToolTipText = "Delete marked files", + .ToolTipText = "Delete selected files", .Image = My.Resources.DeletePic_24, .DisplayStyle = ToolStripItemDisplayStyle.ImageAndText } + BTT_COPY_SELECTED_T = New ToolStripButton With { + .Text = "Copy", + .AutoToolTip = True, + .ToolTipText = "Copy selected files", + .Image = My.Resources.PastePic_32, + .DisplayStyle = ToolStripItemDisplayStyle.ImageAndText + } + BTT_MOVE_SELECTED_T = New ToolStripButton With { + .Text = "Move", + .AutoToolTip = True, + .ToolTipText = "Move selected files", + .Image = My.Resources.CutPic_48, + .DisplayStyle = ToolStripItemDisplayStyle.ImageAndText + } + BTT_FAV_T = New ToolStripButton With { + .Text = "Favorite", + .AutoToolTip = True, + .ToolTipText = "Add checked to Favorite", + .Image = My.Resources.HeartPic_32, + .DisplayStyle = ToolStripItemDisplayStyle.ImageAndText + } + BTT_FEED_ADD_T = New ToolStripDropDownButton With { + .Text = "Add", + .AutoToolTip = True, + .ToolTipText = "Add checked to special feed..." & vbCr & "Right click to add to multiple feeds", + .Image = My.Resources.RSSPic_512, + .DisplayStyle = ToolStripItemDisplayStyle.ImageAndText + } + BTT_SELECT_ALL_T = New ToolStripButton With { + .Text = "All", + .AutoToolTip = True, + .ToolTipText = "Select all", + .DisplayStyle = ToolStripItemDisplayStyle.Text + } + BTT_SELECT_NONE_T = New ToolStripButton With { + .Text = "None", + .AutoToolTip = True, + .ToolTipText = "Select none", + .DisplayStyle = ToolStripItemDisplayStyle.Text + } + BTT_INVERT_T = New ToolStripButton With { + .Text = "Invert", + .AutoToolTip = True, + .ToolTipText = "Invert selection", + .DisplayStyle = ToolStripItemDisplayStyle.Text + } + FILTERS = New FeedFilterCollection BTT_FILTER.Image = My.Resources.FilterPic BTT_FILTER_SIMPLE.Image = My.Resources.FilterPic @@ -189,7 +246,10 @@ Namespace DownloadObjects .ToolTip(RCI.GoTo) = "GoTo (Ctrl+G)" .AddThisToolbar() End With - ToolbarTOP.Items.AddRange({New ToolStripSeparator, BTT_DELETE_SELECTED}) + ToolbarTOP.Items.AddRange({New ToolStripSeparator, BTT_DELETE_SELECTED, + New ToolStripSeparator, BTT_COPY_SELECTED_T, BTT_MOVE_SELECTED_T, + New ToolStripSeparator, BTT_FAV_T, BTT_FEED_ADD_T, + New ToolStripSeparator, BTT_SELECT_ALL_T, BTT_SELECT_NONE_T, BTT_INVERT_T}) With Settings With .Feeds .Load() @@ -200,6 +260,7 @@ Namespace DownloadObjects If Not feed.IsFavorite Then AddNewFeedItem(BTT_LOAD_SPEC, feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_LOAD) AddNewFeedItem(BTT_FEED_ADD_SPEC, feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_ADD) + AddNewFeedItem(BTT_FEED_ADD_T, feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_ADD) AddNewFeedItem(BTT_FEED_ADD_SPEC_REMOVE, feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_ADD_REMOVE) AddNewFeedItem(BTT_FEED_REMOVE_SPEC, feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_REMOVE) AddNewFeedItem(BTT_FEED_DELETE_SPEC, feed, My.Resources.DeletePic_24, AddressOf Feed_SPEC_DELETE) @@ -268,11 +329,11 @@ Namespace DownloadObjects End Sub #End Region #Region "Feeds handlers" - Private Overloads Sub AddNewFeedItem(ByVal Destination As ToolStripMenuItem, ByVal Feed As FeedSpecial, ByVal Image As Image, - ByVal Handler As EventHandler, Optional ByVal Insert As Boolean = False) + Private Overloads Sub AddNewFeedItem(Of T As ToolStripDropDownItem)(ByVal Destination As T, ByVal Feed As FeedSpecial, ByVal Image As Image, + ByVal Handler As EventHandler, Optional ByVal Insert As Boolean = False) AddNewFeedItem(Destination, ToolbarTOP, Feed, Image, Handler, Insert) End Sub - Friend Overloads Shared Function AddNewFeedItem(ByVal Destination As ToolStripMenuItem, ByVal Toolbar As ToolStrip, + Friend Overloads Shared Function AddNewFeedItem(ByVal Destination As ToolStripDropDownItem, ByVal Toolbar As ToolStrip, ByVal Feed As FeedSpecial, ByVal Image As Image, ByVal Handler As EventHandler, Optional ByVal Insert As Boolean = False) As ToolStripMenuItem Dim item As New ToolStripMenuItem(Feed.Name, Image) With {.Tag = Feed} @@ -289,6 +350,7 @@ Namespace DownloadObjects Private Sub Feed_FeedAdded(ByVal Source As FeedSpecialCollection, ByVal Feed As FeedSpecial) AddNewFeedItem(BTT_LOAD_SPEC, Feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_LOAD, True) AddNewFeedItem(BTT_FEED_ADD_SPEC, Feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_ADD, True) + AddNewFeedItem(BTT_FEED_ADD_T, Feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_ADD, True) AddNewFeedItem(BTT_FEED_ADD_SPEC_REMOVE, Feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_ADD_REMOVE, True) AddNewFeedItem(BTT_FEED_REMOVE_SPEC, Feed, My.Resources.RSSPic_512, AddressOf Feed_SPEC_REMOVE, True) AddNewFeedItem(BTT_FEED_DELETE_SPEC, Feed, My.Resources.DeletePic_24, AddressOf Feed_SPEC_DELETE, True) @@ -297,14 +359,15 @@ Namespace DownloadObjects Private Overloads Sub Feed_FeedRemoved(ByVal Source As FeedSpecialCollection, ByVal Feed As FeedSpecial) Feed_FeedRemoved(BTT_LOAD_SPEC, Feed) Feed_FeedRemoved(BTT_FEED_ADD_SPEC, Feed) + Feed_FeedRemoved(BTT_FEED_ADD_T, Feed) Feed_FeedRemoved(BTT_FEED_REMOVE_SPEC, Feed) Feed_FeedRemoved(BTT_FEED_DELETE_SPEC, Feed) Feed_FeedRemoved(BTT_FEED_CLEAR_SPEC, Feed) End Sub - Private Overloads Sub Feed_FeedRemoved(ByVal Destination As ToolStripMenuItem, ByVal Feed As FeedSpecial) + Private Overloads Sub Feed_FeedRemoved(ByVal Destination As ToolStripDropDownItem, ByVal Feed As FeedSpecial) Feed_FeedRemoved(Destination, ToolbarTOP, Feed) End Sub - Friend Overloads Shared Sub Feed_FeedRemoved(ByVal Destination As ToolStripMenuItem, ByVal Toolbar As ToolStrip, ByVal Feed As FeedSpecial) + Friend Overloads Shared Sub Feed_FeedRemoved(ByVal Destination As ToolStripDropDownItem, ByVal Toolbar As ToolStrip, ByVal Feed As FeedSpecial) Try With Destination ControlInvokeFast(Toolbar, .Self, @@ -594,7 +657,7 @@ Namespace DownloadObjects End Sub #End Region #Region "Move/Copy" - Private Sub BTT_COPY_MOVE_TO_Click(sender As Object, e As EventArgs) Handles BTT_COPY_TO.Click, BTT_MOVE_TO.Click + Private Sub BTT_COPY_MOVE_TO_Click(sender As Object, e As EventArgs) Handles BTT_COPY_TO.Click, BTT_MOVE_TO.Click, BTT_COPY_SELECTED_T.Click, BTT_MOVE_SELECTED_T.Click MoveCopyFiles(True, sender, Nothing, Nothing) End Sub Private Sub BTT_COPY_MOVE_SPEC_TO_Click(sender As Object, e As EventArgs) Handles BTT_COPY_SPEC_TO.Click, BTT_MOVE_SPEC_TO.Click @@ -604,7 +667,7 @@ Namespace DownloadObjects ByVal FeedMediaData As FeedMedia, Optional ByVal GetChecked As Boolean = True) As Boolean Dim MsgTitle$ = "Copy/Move checked files" Try - Dim isCopy As Boolean = Not Sender Is Nothing AndAlso (Sender Is BTT_COPY_TO OrElse Sender Is BTT_COPY_SPEC_TO) + Dim isCopy As Boolean = Not Sender Is Nothing AndAlso (Sender Is BTT_COPY_TO OrElse Sender Is BTT_COPY_SPEC_TO OrElse Sender Is BTT_COPY_SELECTED_T) Dim moveOptions As FeedMoveCopyTo = Nothing Dim ff As SFile = Nothing, ffInit As SFile = Nothing, df As SFile = Nothing Dim data As IEnumerable(Of UserMediaD) = Nothing @@ -1004,7 +1067,7 @@ Namespace DownloadObjects End Sub #End Region #Region "Add remove fav spec" - Private Sub BTT_FEED_ADD_FAV_Click(sender As Object, e As EventArgs) Handles BTT_FEED_ADD_FAV.Click, BTT_FEED_ADD_FAV_REMOVE.Click + Private Sub BTT_FEED_ADD_FAV_Click(sender As Object, e As EventArgs) Handles BTT_FEED_ADD_FAV.Click, BTT_FEED_ADD_FAV_REMOVE.Click, BTT_FAV_T.Click Dim m As IEnumerable(Of UserMediaD) = GetCheckedMedia() If m.ListExists Then Settings.Feeds.Favorite.Add(m) @@ -1018,6 +1081,9 @@ Namespace DownloadObjects If FeedMode = FeedModes.Special Then FeedRemoveCheckedMedia(m, {FeedSpecial.FavoriteName}.ToList,,, False) End If End Sub + Private Sub BTT_FEED_ADD_T_MouseDown(sender As Object, e As MouseEventArgs) Handles BTT_FEED_ADD_T.MouseDown + If e.Button = MouseButtons.Right Then BTT_FEED_ADD_SPEC_Click(sender, e) + End Sub Private Sub BTT_FEED_ADD_SPEC_Click(sender As Object, e As EventArgs) Handles BTT_FEED_ADD_SPEC.Click, BTT_FEED_ADD_SPEC_REMOVE.Click Dim c As IEnumerable(Of UserMediaD) = GetCheckedMedia() If c.ListExists Then @@ -1276,10 +1342,10 @@ Namespace DownloadObjects End Try End Sub #End Region - Private Sub BTT_CHECK_ALL_NONE_Click(sender As Object, e As EventArgs) Handles BTT_CHECK_ALL.Click, BTT_CHECK_NONE.Click, BTT_CHECK_INVERT.Click + Private Sub BTT_CHECK_ALL_NONE_Click(sender As Object, e As EventArgs) Handles BTT_CHECK_ALL.Click, BTT_CHECK_NONE.Click, BTT_CHECK_INVERT.Click, BTT_SELECT_ALL_T.Click, BTT_SELECT_NONE_T.Click, BTT_INVERT_T.Click Try - Dim checked As Boolean = sender Is BTT_CHECK_ALL - Dim isInvert As Boolean = sender Is BTT_CHECK_INVERT + Dim checked As Boolean = sender Is BTT_CHECK_ALL Or sender Is BTT_SELECT_ALL_T + Dim isInvert As Boolean = sender Is BTT_CHECK_INVERT Or sender Is BTT_INVERT_T ControlInvokeFast(TP_DATA, Sub() With TP_DATA If .Controls.Count > 0 Then diff --git a/SCrawler/Download/Feed/FeedMedia.vb b/SCrawler/Download/Feed/FeedMedia.vb index 93044a1..cb471bb 100644 --- a/SCrawler/Download/Feed/FeedMedia.vb +++ b/SCrawler/Download/Feed/FeedMedia.vb @@ -204,7 +204,17 @@ Namespace DownloadObjects End With If Not imgFile.Exists Then Settings.Cache.Validate() - If GetWebFile(Media.Data.URL, imgFile, EDP.None) AndAlso imgFile.Exists Then File = UserImage.ConvertWebp(imgFile, Nothing) + If GetWebFile(Media.Data.URL, imgFile, EDP.None) AndAlso imgFile.Exists Then + File = UserImage.ConvertWebp(imgFile, Nothing) + ElseIf Not IsNothing(Media.Data.Object) AndAlso TypeOf Media.Data.Object Is IThumbList AndAlso + DirectCast(Media.Data.Object, IThumbList).Thumbs.ListExists Then + With DirectCast(Media.Data.Object, IThumbList).Thumbs + For Each __tmpUrl$ In .Self + If GetWebFile(__tmpUrl, imgFile, EDP.None) AndAlso imgFile.Exists Then _ + File = UserImage.ConvertWebp(imgFile, Nothing) : Exit For + Next + End With + End If Else File = imgFile End If @@ -225,7 +235,7 @@ Namespace DownloadObjects End If End If - If File.Exists Then + If File.Exists Or IsSubscription Then Information = $"Type: {IIf(ExtractText, UserMedia.Types.Text.ToString, Media.Data.Type.ToString)}" Information.StringAppendLine($"File: {File.File}") Information.StringAppendLine($"Address: {File}") @@ -256,7 +266,7 @@ Namespace DownloadObjects Dim webpConverted As Boolean = False Dim isWebp As Boolean = False tmpMediaFile = UserImage.ConvertWebp(tmpMediaFile, Nothing,,, webpConverted) - If tmpMediaFile.IsEmptyString Then Throw New ArgumentNullException With {.HelpLink = 1} + If tmpMediaFile.IsEmptyString And Not IsSubscription Then Throw New ArgumentNullException With {.HelpLink = 1} Try For kConv As Byte = 0 To 1 If kConv = 1 Then tmpMediaFile = UserImage.ConvertWebp(tmpMediaFile, Nothing, True, isWebp, webpConverted) diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 4a7c0b4..7030300 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + +