mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-24 12:42:18 +00:00
Compare commits
1 Commits
2026.2.14.
...
2026.3.23.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be97752b7c |
25
Changelog.md
25
Changelog.md
@@ -2,13 +2,34 @@
|
||||
- [ffmpeg](https://github.com/AAndyProgram/SCrawler/wiki/Settings#ffmpeg)
|
||||
- x64 version - [release](https://github.com/GyanD/codexffmpeg/releases/tag/5.1.2); [zip](https://github.com/GyanD/codexffmpeg/releases/download/5.1.2/ffmpeg-5.1.2-full_build.zip); **version `5.1.2-full_build-www.gyan.dev`**
|
||||
- x86 version - [release](https://github.com/yt-dlp/FFmpeg-Builds/releases/tag/autobuild-2022-11-30-12-57); [zip](https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-2022-11-30-12-57/ffmpeg-N-109274-gd7a5f068c2-win32-gpl.zip); **version `N-109457-geeb280f351-20221226`**
|
||||
- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.6**
|
||||
- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.02.04.233607**
|
||||
- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.9**
|
||||
- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.03.17**
|
||||
- [Deno](https://github.com/AAndyProgram/SCrawler/wiki/Settings#deno) - latest *(`2.0.0` or higher)*
|
||||
- [OF-Scraper](https://github.com/AAndyProgram/SCrawler/wiki/Settings#of-scraper) - **3.12.9** ([release](https://github.com/datawhores/OF-Scraper/releases/tag/3.12.9))
|
||||
|
||||
# 2026
|
||||
|
||||
## 2026.3.23.0
|
||||
|
||||
*2026-03-23*
|
||||
|
||||
- Added
|
||||
- Sites:
|
||||
- Bluesky:
|
||||
- download models (user options), like in Twitter
|
||||
- `did` handle for recognizing
|
||||
- PornHub: `M3U8` download functionality update
|
||||
- xHamster: add `webm` to `jpg` correct conversion
|
||||
- Minor improvements
|
||||
- Updated
|
||||
- gallery-dl up to version **1.31.9**
|
||||
- yt-dlp up to version **2026.03.17**
|
||||
- Fixed
|
||||
- Sites:
|
||||
- TikTok: gallery-dl downloads stories slowly when the user has no stories
|
||||
- Twitter: downloading a single media file (standalone downloader) fails
|
||||
- Minor bugs
|
||||
|
||||
## 2026.2.14.0
|
||||
|
||||
*2026-02-14*
|
||||
|
||||
@@ -42,5 +42,9 @@ Namespace API.Base
|
||||
u.DownloadTextSpecialFolder = DownloadTextSpecialFolder
|
||||
End If
|
||||
End Sub
|
||||
Friend Overridable Sub Apply(ByRef u As UserDataBase)
|
||||
ApplyBase(u)
|
||||
If TypeOf u Is IPSite And TypeOf Me Is IPSite Then DirectCast(u, IPSite).QueryString = DirectCast(Me, IPSite).QueryString
|
||||
End Sub
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -31,10 +31,6 @@ Namespace API.Base
|
||||
MyBase.New(s)
|
||||
DisableBase()
|
||||
End Sub
|
||||
Friend Overridable Sub Apply(ByRef u As IPSite)
|
||||
ApplyBase(u)
|
||||
u.QueryString = QueryString
|
||||
End Sub
|
||||
Protected Overridable Sub DisableBase()
|
||||
_ApplyBase_Name = False
|
||||
_ApplyBase_Text = False
|
||||
|
||||
@@ -360,8 +360,8 @@ Namespace API.Base
|
||||
Me._UserSiteName = _UserSiteName
|
||||
End Set
|
||||
End Property
|
||||
Protected Sub UserSiteNameUpdate(ByVal NewName As String)
|
||||
If Not NewName.IsEmptyString And (UserSiteName.IsEmptyString Or Settings.UpdateUserSiteNameEveryTime) Then UserSiteName = NewName
|
||||
Protected Sub UserSiteNameUpdate(ByVal NewName As String, Optional ByVal Force As Boolean = False)
|
||||
If Not NewName.IsEmptyString And (UserSiteName.IsEmptyString Or Settings.UpdateUserSiteNameEveryTime Or Force) Then UserSiteName = NewName
|
||||
End Sub
|
||||
Friend ReadOnly Property UserModel As UsageModel Implements IUserData.UserModel
|
||||
Get
|
||||
@@ -1030,7 +1030,7 @@ BlockNullPicture:
|
||||
End Try
|
||||
End Sub
|
||||
Private Sub UpdateUserInformation_Ex()
|
||||
If _ForceSaveUserInfoOnException Then UpdateUserInformation()
|
||||
If _ForceSaveUserInfo Then UpdateUserInformation()
|
||||
End Sub
|
||||
Friend Overridable Overloads Sub UpdateUserInformation() Implements IUserData.UpdateUserInformation
|
||||
UpdateUserInformation(False)
|
||||
@@ -1207,7 +1207,6 @@ BlockNullPicture:
|
||||
Protected UseClientTokens As Boolean = False
|
||||
Protected _ForceSaveUserData As Boolean = False
|
||||
Protected _ForceSaveUserInfo As Boolean = False
|
||||
Protected _ForceSaveUserInfoOnException As Boolean = False
|
||||
Private _DownloadInProgress As Boolean = False
|
||||
Private _EnvirUserExists As Boolean
|
||||
Private _EnvirUserSuspended As Boolean
|
||||
@@ -1227,7 +1226,6 @@ BlockNullPicture:
|
||||
_DescriptionEveryTime = Settings.UpdateUserDescriptionEveryTime
|
||||
_ForceSaveUserData = False
|
||||
_ForceSaveUserInfo = False
|
||||
_ForceSaveUserInfoOnException = False
|
||||
_EnvirUserExists = UserExists
|
||||
_EnvirUserSuspended = UserSuspended
|
||||
_EnvirCreatedByChannel = CreatedByChannel
|
||||
@@ -1244,8 +1242,8 @@ BlockNullPicture:
|
||||
Select Case Caller
|
||||
Case NameOf(UserExists) : If Not _EnvirUserExists = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True
|
||||
Case NameOf(UserSuspended) : If Not _EnvirUserSuspended = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True
|
||||
Case NameOf(NameTrue) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
|
||||
Case NameOf(ID) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
|
||||
Case NameOf(NameTrue) : _EnvirChanged = True : _ForceSaveUserInfo = True
|
||||
Case NameOf(ID) : _EnvirChanged = True : _ForceSaveUserInfo = True
|
||||
Case Else : _EnvirChanged = True
|
||||
End Select
|
||||
End If
|
||||
@@ -1843,7 +1841,8 @@ BlockNullPicture:
|
||||
|
||||
updateDownCount(False)
|
||||
|
||||
v.File = DownloadContentDefault_ConvertWebp(ChangeFileNameByProvider(f, v), postProcessWebp)
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
v.File = DownloadContentDefault_ConvertWebp(v, postProcessWebp)
|
||||
v.State = UStates.Downloaded
|
||||
DownloadContentDefault_PostProcessing(v, f, Token)
|
||||
If UseMD5Comparison And (v.Type = UTypes.GIF Or v.Type = UTypes.Picture) Then
|
||||
@@ -1943,7 +1942,11 @@ stxt:
|
||||
End Function
|
||||
Protected Overridable Sub DownloadContentDefault_PostProcessing(ByRef m As UserMedia, ByVal File As SFile, ByVal Token As CancellationToken)
|
||||
End Sub
|
||||
Protected Overridable Function DownloadContentDefault_ConvertWebp(ByVal WebpFile As SFile, ByVal Process As Boolean) As SFile
|
||||
Protected Overridable Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
|
||||
Return DownloadContentDefault_ConvertWebp_Impl(m, Process)
|
||||
End Function
|
||||
Protected Overridable Function DownloadContentDefault_ConvertWebp_Impl(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
|
||||
Dim WebpFile As SFile = m.File
|
||||
Dim f As SFile = WebpFile
|
||||
If Process AndAlso f.Exists Then
|
||||
f.Path = $"{f.PathWithSeparator}Sources"
|
||||
@@ -1959,6 +1962,24 @@ stxt:
|
||||
End If
|
||||
Return f
|
||||
End Function
|
||||
Protected Function DownloadContentDefault_ConvertWebp_TestImg(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
|
||||
If m.Type = UTypes.Picture And Settings.DownloadNativeImageFormat Then
|
||||
Using testImg As New UserImage.ImageRenderer2(m.File, EDP.ReturnValue)
|
||||
If testImg.IsWebP Then
|
||||
Dim f As SFile = m.File
|
||||
If f.Extension.IsEmptyString OrElse Not f.Extension = UserImage.ExtWebp Then
|
||||
f.Extension = UserImage.ExtWebp
|
||||
f = SFile.Rename(m.File, f,, EDP.ReturnValue).IfNullOrEmpty(m.File)
|
||||
End If
|
||||
m.File = f
|
||||
Return DownloadContentDefault_ConvertWebp_Impl(m, True).IfNullOrEmpty(f)
|
||||
End If
|
||||
End Using
|
||||
ElseIf Process Then
|
||||
Return DownloadContentDefault_ConvertWebp_Impl(m, Process)
|
||||
End If
|
||||
Return m.File
|
||||
End Function
|
||||
Protected Overridable Function DownloadContentDefault_ProcessDownloadException() As Boolean
|
||||
Return True
|
||||
End Function
|
||||
|
||||
44
SCrawler/API/Bluesky/EditorExchangeOptions.vb
Normal file
44
SCrawler/API/Bluesky/EditorExchangeOptions.vb
Normal file
@@ -0,0 +1,44 @@
|
||||
' Copyright (C) Andy https://github.com/AAndyProgram
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports SCrawler.API.Base
|
||||
Imports SCrawler.Plugin.Attributes
|
||||
Namespace API.Bluesky
|
||||
Friend Class EditorExchangeOptions : Inherits Base.EditorExchangeOptionsBase
|
||||
Friend Overrides Property SiteKey As String = BlueskySiteKey
|
||||
<PSetting(NameOf(SiteSettings.DownloadModelMedia), NameOf(MySettings), Address:=SettingAddress.User)>
|
||||
Friend Overridable Property DownloadModelMedia As Boolean = False
|
||||
<PSetting(NameOf(SiteSettings.DownloadModelProfile), NameOf(MySettings), Address:=SettingAddress.User)>
|
||||
Friend Overridable Property DownloadModelProfile As Boolean = False
|
||||
Private ReadOnly Property MySettings As Object
|
||||
Friend Sub New(ByVal s As SiteSettings)
|
||||
DownloadModelMedia = s.DownloadModelMedia.Value
|
||||
DownloadModelProfile = s.DownloadModelProfile.Value
|
||||
MySettings = s
|
||||
End Sub
|
||||
Friend Sub New(ByVal u As UserData)
|
||||
DownloadModelMedia = u.DownloadModelMedia
|
||||
DownloadModelProfile = u.DownloadModelProfile
|
||||
MySettings = u.HOST.Source
|
||||
End Sub
|
||||
Friend Overrides Sub Apply(ByRef u As UserDataBase)
|
||||
MyBase.Apply(u)
|
||||
If Not DownloadModelMedia And Not DownloadModelProfile Then
|
||||
DownloadModelMedia = True
|
||||
ElseIf DownloadModelMedia Then
|
||||
DownloadModelProfile = False
|
||||
Else
|
||||
DownloadModelMedia = False
|
||||
End If
|
||||
With DirectCast(u, UserData)
|
||||
.DownloadModelMedia = DownloadModelMedia
|
||||
.DownloadModelProfile = DownloadModelProfile
|
||||
End With
|
||||
End Sub
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -26,6 +26,10 @@ Namespace API.Bluesky
|
||||
<PXML> Friend ReadOnly Property TokenUpdateTime As PropertyValue
|
||||
<PropertyOption(ControlText:="Token update", ControlToolTip:="Token refresh interval (in minutes)." & vbCr & "Default: 120.", IsAuth:=True), PXML, PClonable, HiddenControl>
|
||||
Friend ReadOnly Property TokenRefreshInterval As PropertyValue
|
||||
<PropertyOption(ControlText:="Download model 'Media'", ControlToolTip:="Parse the 'Media' block", Category:=DeclaredNames.CAT_UserDefs), PXML, PClonable>
|
||||
Friend ReadOnly Property DownloadModelMedia As PropertyValue
|
||||
<PropertyOption(ControlText:="Download model 'Profile'", ControlToolTip:="Parse the 'Posts' block", Category:=DeclaredNames.CAT_UserDefs), PXML, PClonable>
|
||||
Friend ReadOnly Property DownloadModelProfile As PropertyValue
|
||||
Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
|
||||
MyBase.New("Bluesky", "bsky.app", AccName, Temp, My.Resources.SiteResources.BlueskyIcon_32, My.Resources.SiteResources.BlueskyPic_32)
|
||||
|
||||
@@ -38,11 +42,14 @@ Namespace API.Bluesky
|
||||
TokenUpdateTime = New PropertyValue(Now.AddYears(-1))
|
||||
TokenRefreshInterval = New PropertyValue(120)
|
||||
|
||||
DownloadModelMedia = New PropertyValue(True)
|
||||
DownloadModelProfile = New PropertyValue(False)
|
||||
|
||||
_AllowUserAgentUpdate = False
|
||||
UrlPatternUser = "https://bsky.app/profile/{0}"
|
||||
ImageVideoContains = "bsky.app"
|
||||
UserRegex = RParams.DMS("bsky.app/profile/([^/\?]+)", 1, EDP.ReturnValue)
|
||||
UserOptionsType = GetType(EditorExchangeOptionsBase)
|
||||
UserOptionsType = GetType(EditorExchangeOptions)
|
||||
End Sub
|
||||
Protected Overrides Function UserOptionsValid(ByVal Options As Object) As Boolean
|
||||
Return DirectCast(Options, EditorExchangeOptionsBase).SiteKey = BlueskySiteKey
|
||||
@@ -96,5 +103,10 @@ Namespace API.Bluesky
|
||||
_TokenUpdating = False
|
||||
End Try
|
||||
End Function
|
||||
Friend Overrides Function IsMyUser(ByVal UserURL As String) As ExchangeOptions
|
||||
Dim e As ExchangeOptions = MyBase.IsMyUser(UserURL)
|
||||
If Not e.UserName.IsEmptyString AndAlso e.UserName.StartsWith("did:") Then e.UserName = e.UserName.Replace(":", "@")
|
||||
Return e
|
||||
End Function
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -16,6 +16,11 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types
|
||||
Imports UStates = SCrawler.API.Base.UserMedia.States
|
||||
Namespace API.Bluesky
|
||||
Friend Class UserData : Inherits UserDataBase
|
||||
#Region "XML names"
|
||||
Private Const Name_DownloadModelMedia As String = "DownloadModelMedia"
|
||||
Private Const Name_DownloadModelProfile As String = "DownloadModelProfile"
|
||||
Private Const Name_ForceParseProfileInfo As String = "ForceParseProfileInfo"
|
||||
#End Region
|
||||
#Region "Declarations"
|
||||
Private ReadOnly Property MySettings As SiteSettings
|
||||
Get
|
||||
@@ -28,16 +33,45 @@ Namespace API.Bluesky
|
||||
End Get
|
||||
End Property
|
||||
Private ReadOnly _TmpPosts2 As List(Of String)
|
||||
Friend Property DownloadModelMedia As Boolean = True
|
||||
Friend Property DownloadModelProfile As Boolean = False
|
||||
Private Property ForceParseProfileInfo As Boolean = False
|
||||
#End Region
|
||||
#Region "Loader"
|
||||
Private Sub UpdateUserOptions()
|
||||
If ID.IsEmptyString AndAlso Not Name.IsEmptyString AndAlso Name.StartsWith("did@") Then
|
||||
NameTrue = Name.Replace("@", ":")
|
||||
ID = NameTrue
|
||||
ForceParseProfileInfo = True
|
||||
End If
|
||||
End Sub
|
||||
Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
|
||||
With Container
|
||||
If Loading Then
|
||||
If .Contains(Name_DownloadModelMedia) Then
|
||||
DownloadModelMedia = .Value(Name_DownloadModelMedia).FromXML(Of Boolean)(True)
|
||||
DownloadModelProfile = .Value(Name_DownloadModelProfile).FromXML(Of Boolean)(False)
|
||||
Else
|
||||
DownloadModelMedia = ParseUserMediaOnly
|
||||
DownloadModelProfile = Not ParseUserMediaOnly
|
||||
End If
|
||||
ForceParseProfileInfo = .Value(Name_ForceParseProfileInfo).FromXML(Of Boolean)(False)
|
||||
Else
|
||||
If ID.IsEmptyString Then
|
||||
UpdateUserOptions()
|
||||
.Value(Name_UserID) = ID
|
||||
End If
|
||||
.Add(Name_DownloadModelMedia, DownloadModelMedia.BoolToInteger)
|
||||
.Add(Name_DownloadModelProfile, DownloadModelProfile.BoolToInteger)
|
||||
.Add(Name_ForceParseProfileInfo, ForceParseProfileInfo.BoolToInteger)
|
||||
End If
|
||||
End With
|
||||
End Sub
|
||||
Friend Overrides Function ExchangeOptionsGet() As Object
|
||||
Return New EditorExchangeOptionsBase(Me) With {.SiteKey = BlueskySiteKey}
|
||||
Return New EditorExchangeOptions(Me)
|
||||
End Function
|
||||
Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object)
|
||||
If Not Obj Is Nothing AndAlso TypeOf Obj Is EditorExchangeOptionsBase AndAlso
|
||||
DirectCast(Obj, EditorExchangeOptionsBase).SiteKey = BlueskySiteKey Then DirectCast(Obj, EditorExchangeOptionsBase).ApplyBase(Me)
|
||||
If Not Obj Is Nothing AndAlso TypeOf Obj Is EditorExchangeOptions Then DirectCast(Obj, EditorExchangeOptions).Apply(Me)
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Initializer"
|
||||
@@ -66,6 +100,13 @@ Namespace API.Bluesky
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
_TmpPosts2.Clear()
|
||||
Try
|
||||
If Not DownloadModelMedia And Not DownloadModelProfile Then
|
||||
DownloadModelMedia = True
|
||||
ElseIf DownloadModelMedia Then
|
||||
DownloadModelProfile = False
|
||||
Else
|
||||
DownloadModelMedia = False
|
||||
End If
|
||||
If Not CBool(MySettings.CookiesEnabled.Value) Then Responser.Cookies.Clear()
|
||||
UpdateToken(, True)
|
||||
_TokenUpdateCount = 0
|
||||
@@ -79,7 +120,7 @@ Namespace API.Bluesky
|
||||
Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
If Not IsSavedPosts And ID.IsEmptyString Then GetProfileInfo(Token)
|
||||
If (Not IsSavedPosts And ID.IsEmptyString) Or ForceParseProfileInfo Then GetProfileInfo(Token)
|
||||
If Not IsSavedPosts And ID.IsEmptyString Then Throw New ArgumentNullException("ID", "ID is null")
|
||||
If UpdateToken() Then
|
||||
Dim nextCursor$ = String.Empty
|
||||
@@ -91,7 +132,9 @@ Namespace API.Bluesky
|
||||
n = {"bookmarks"}
|
||||
p = {"item"}
|
||||
Else
|
||||
URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter=posts_and_author_threads&includePins=false&limit=99"
|
||||
'posts_and_author_threads
|
||||
'posts_with_media
|
||||
URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter={IIf(DownloadModelMedia, "posts_with_media", "posts_and_author_threads")}&includePins=false&limit=99"
|
||||
If Not Cursor.IsEmptyString Then URL &= $"&cursor={SymbolsConverter.ASCII.EncodeSymbolsOnly(Cursor)}"
|
||||
n = {"feed"}
|
||||
p = {"post"}
|
||||
@@ -106,7 +149,7 @@ Namespace API.Bluesky
|
||||
If .ListExists Then
|
||||
For Each post As EContainer In .Self
|
||||
With post(p)
|
||||
c = DefaultParser(.Self,, nextCursor)
|
||||
c = DefaultParser(.Self) ',, nextCursor)
|
||||
Select Case c
|
||||
Case CInt(DateResult.Skip) * -1 : Continue For
|
||||
Case CInt(DateResult.Exit) * -1 : Exit Sub
|
||||
@@ -238,6 +281,7 @@ Namespace API.Bluesky
|
||||
#Region "GetProfileInfo"
|
||||
Private Sub GetProfileInfo(ByVal Token As CancellationToken)
|
||||
Try
|
||||
If ForceParseProfileInfo Then ForceParseProfileInfo = False : _ForceSaveUserInfo = True
|
||||
If UpdateToken() Then
|
||||
Dim r$ = Responser.GetResponse($"https://bsky.social/xrpc/app.bsky.actor.getProfile?actor={ID.IfNullOrEmpty(NameTrue)}")
|
||||
TokenUpdateCountReset()
|
||||
@@ -344,6 +388,9 @@ Namespace API.Bluesky
|
||||
Protected Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile
|
||||
Return M3U8.Download(URL, DestinationFile, Token, Progress, Not IsSingleObjectDownload)
|
||||
End Function
|
||||
Protected Overrides Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
|
||||
Return DownloadContentDefault_ConvertWebp_TestImg(m, Process)
|
||||
End Function
|
||||
#End Region
|
||||
#Region "DownloadSingleObject"
|
||||
Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
|
||||
|
||||
@@ -702,7 +702,7 @@ Namespace API.Instagram
|
||||
If Not IsSavedPosts Then
|
||||
If _UseGQL And Cursor.IsEmptyString And Not Section = Sections.SavedPosts Then UpdateTokens(True)
|
||||
If ID.IsEmptyString Or __idIsEmpty Or Not IsVerifiedProfile_Checked Then GetUserData(Token)
|
||||
If ID.IsEmptyString Then UserExists = False : _ForceSaveUserInfoOnException = True : Throw New Plugin.ExitException("can't get user ID")
|
||||
If ID.IsEmptyString Then UserExists = False : _ForceSaveUserInfo = True : Throw New Plugin.ExitException("can't get user ID")
|
||||
If ForceUpdateUserName Then GetUserNameById()
|
||||
If ForceUpdateUserInfo Then GetUserData(Token)
|
||||
End If
|
||||
|
||||
@@ -22,7 +22,7 @@ Namespace API.PornHub
|
||||
If Not r.IsEmptyString Then
|
||||
Dim files As List(Of Sizes) = RegexFields(Of Sizes)(r, {Regex_M3U8_FilesList}, {1, 2}, EDP.ReturnValue)
|
||||
Dim file$
|
||||
If files.ListExists Then files.RemoveAll(Function(f) f.Value = 0 Or (Not DownloadUHD And f.Value > 1080))
|
||||
If files.ListExists Then files.RemoveAll(Function(f) f.Value = 0 Or (Not DownloadUHD And f.Value > 1080) Or (f.Data.IsEmptyString OrElse f.Data.Contains("""")))
|
||||
If files.ListExists Then
|
||||
files.Sort()
|
||||
file = files(0).Data
|
||||
|
||||
@@ -258,7 +258,6 @@ Namespace API.PornHub
|
||||
Private _PageVideosRepeat As Integer = 0
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
Try
|
||||
UpdateM3U8URLS = False
|
||||
PlaylistToken = String.Empty
|
||||
Responser.ResetStatus()
|
||||
_PageVideosRepeat = 0
|
||||
@@ -793,31 +792,29 @@ Namespace API.PornHub
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download content"
|
||||
Private UpdateM3U8URLS As Boolean = False
|
||||
Private UpdateM3U8URLS_Error As Boolean = False
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
Try : DownloadContentDefault(Token) : Finally : UpdateM3U8URLS = False : End Try
|
||||
DownloadContentDefault(Token)
|
||||
End Sub
|
||||
Protected Overloads Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile,
|
||||
ByVal Token As CancellationToken) As SFile
|
||||
UpdateM3U8URLS_Error = False
|
||||
Return DownloadM3U8(URL, Media, DestinationFile, Token, UpdateM3U8URLS)
|
||||
Return DownloadM3U8(URL, Media, DestinationFile, Token, 0)
|
||||
End Function
|
||||
Private Overloads Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile,
|
||||
ByVal Token As CancellationToken, ByVal Second As Boolean) As SFile
|
||||
ByVal Token As CancellationToken, ByVal Round As Integer) As SFile
|
||||
Const MaxRound% = 2
|
||||
Try
|
||||
If Second Then
|
||||
If Round > 0 Then
|
||||
Dim r$ = Responser.Curl(Media.URL_BASE,, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then Media.URL = CreateVideoURL(r).IfNullOrEmpty(URL) : URL = Media.URL
|
||||
End If
|
||||
Dim f As SFile = M3U8.Download(URL, Responser, DestinationFile, DownloadUHD, Token, Progress, Not IsSingleObjectDownload)
|
||||
If Not f.Exists And Not Second Then UpdateM3U8URLS = True : f = DownloadM3U8(URL, Media, DestinationFile, Token, True)
|
||||
If Not f.Exists Then f = Nothing
|
||||
If f.IsEmptyString And Round < MaxRound Then f = DownloadM3U8(URL, Media, DestinationFile, Token, Round + 1)
|
||||
Return f
|
||||
Catch ex As Exception
|
||||
If Not UpdateM3U8URLS_Error Then
|
||||
UpdateM3U8URLS_Error = True
|
||||
If Round < MaxRound Then
|
||||
Thread.Sleep(1000)
|
||||
Return DownloadM3U8(URL, Media, DestinationFile, Token, True)
|
||||
Return DownloadM3U8(URL, Media, DestinationFile, Token, Round + 1)
|
||||
End If
|
||||
Return Nothing
|
||||
End Try
|
||||
@@ -917,7 +914,6 @@ Namespace API.PornHub
|
||||
#End Region
|
||||
#Region "DownloadSingleObject"
|
||||
Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
|
||||
UpdateM3U8URLS = False
|
||||
_TempMediaList.Add(New UserMedia(Data.URL, UTypes.VideoPre))
|
||||
ReparseVideo(Token, True, Data)
|
||||
End Sub
|
||||
|
||||
@@ -44,7 +44,7 @@ Namespace API.PornHub
|
||||
DownloadGifs = Not v = CheckState.Unchecked
|
||||
MySettings = s
|
||||
End Sub
|
||||
Friend Overrides Sub Apply(ByRef u As IPSite)
|
||||
Friend Overrides Sub Apply(ByRef u As UserDataBase)
|
||||
MyBase.Apply(u)
|
||||
With DirectCast(u, UserData)
|
||||
.DownloadUHD = DownloadUHD
|
||||
|
||||
@@ -35,7 +35,7 @@ Namespace API.ThisVid
|
||||
DifferentFolders = u.DifferentFolders
|
||||
MySettings = u.HOST.Source
|
||||
End Sub
|
||||
Friend Overrides Sub Apply(ByRef u As IPSite)
|
||||
Friend Overrides Sub Apply(ByRef u As UserDataBase)
|
||||
MyBase.Apply(u)
|
||||
With DirectCast(u, UserData)
|
||||
.DownloadPublic = DownloadPublic
|
||||
|
||||
@@ -184,7 +184,6 @@ Namespace API.ThreadsNet
|
||||
If uex.UserNotFound Then
|
||||
UserExists = False
|
||||
_ForceSaveUserInfo = True
|
||||
_ForceSaveUserInfoOnException = True
|
||||
ElseIf ThrowEx Then
|
||||
Throw New ExitException(uex.ErrMessage) With {.SimpleLogLine = True}
|
||||
Else
|
||||
@@ -404,7 +403,6 @@ Namespace API.ThreadsNet
|
||||
If _IdChanged Then
|
||||
If Not idStr.IsEmptyString Then UserDescriptionUpdate(idStr, True, True, True)
|
||||
_ForceSaveUserInfo = True
|
||||
_ForceSaveUserInfoOnException = True
|
||||
End If
|
||||
End If
|
||||
Return Valid
|
||||
|
||||
@@ -192,10 +192,14 @@ Namespace API.TikTok
|
||||
UserCache.DisposeIfReady(False)
|
||||
UserCache = Nothing
|
||||
End Sub
|
||||
Private StoryStatus As Boolean = True
|
||||
Private StoryStatusFound As Boolean = False
|
||||
Protected Overloads Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
ValidateCache()
|
||||
StoryStatus = True
|
||||
StoryStatusFound = False
|
||||
If GetTimeline Then DownloadDataF(Sections.Timeline, Token)
|
||||
If GetStoriesUser Then DownloadDataF(Sections.UserStories, Token)
|
||||
If GetStoriesUser And StoryStatus Then DownloadDataF(Sections.UserStories, Token)
|
||||
If GetReposts Then DownloadDataF(Sections.Reposts, Token)
|
||||
End Sub
|
||||
Protected Overloads Sub DownloadDataF(ByVal Section As Sections, ByVal Token As CancellationToken)
|
||||
@@ -271,8 +275,8 @@ Namespace API.TikTok
|
||||
Else
|
||||
.TempPostsList = New List(Of String)
|
||||
End If
|
||||
.Execute(CreateGDLCommand(URL, gdlCmd))
|
||||
If Not PhotosDownloaded Then _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
|
||||
.Execute(CreateGDLCommand(URL, gdlCmd,,,, CObj(IIf(Section = Sections.Timeline, dateAfter, Nothing))))
|
||||
If Not PhotosDownloaded Then _ForceSaveUserInfo = True
|
||||
PhotosDownloaded = True
|
||||
End With
|
||||
End Using
|
||||
@@ -436,6 +440,10 @@ Namespace API.TikTok
|
||||
End If
|
||||
End With
|
||||
Else
|
||||
If Section = Sections.Timeline And Not StoryStatusFound And j.ContainsF({0, "webapp.user-detail", "userInfo", "user"}, "UserStoryStatus") Then
|
||||
StoryStatusFound = True
|
||||
StoryStatus = j.ItemF({0, "webapp.user-detail", "userInfo", "user"}, "UserStoryStatus").Value.FromXML(Of Boolean)(False)
|
||||
End If
|
||||
With j.ItemF({0, "webapp.video-detail", "itemInfo", "itemStruct"})
|
||||
If .ListExists Then
|
||||
postID = .Value("id")
|
||||
@@ -604,13 +612,17 @@ Namespace API.TikTok
|
||||
#End Region
|
||||
#Region "GDL Support"
|
||||
Private Function CreateGDLCommand(ByVal URL As String, Optional ByVal SectionCommand As String = Nothing,
|
||||
Optional ByVal IsDownload As Boolean = False, Optional ByVal Output As SFile = Nothing) As String
|
||||
Optional ByVal IsDownload As Boolean = False, Optional ByVal Output As SFile = Nothing,
|
||||
Optional ByVal DateBefore As Date? = Nothing, Optional ByVal DateAfter As Date? = Nothing) As String
|
||||
Dim command$ = $"""{Settings.GalleryDLFile}"" "
|
||||
If Not IsDownload Then
|
||||
command &= "--verbose --no-download --no-skip --write-pages "
|
||||
Else
|
||||
command &= $"--dest ""{Output.PathNoSeparator}"" "
|
||||
End If
|
||||
If DateBefore.HasValue Or DateAfter.HasValue Then
|
||||
With If(DateAfter, DateBefore).Value : command &= $"--filter ""date {IIf(DateAfter.HasValue, ">", "<")} datetime({ .Year}, { .Month}, { .Day}) or terminate()"" " : End With
|
||||
End If
|
||||
If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then command &= "--no-mtime "
|
||||
command &= $"{SectionCommand} {URL}"
|
||||
Return command
|
||||
|
||||
@@ -523,7 +523,7 @@ Namespace API.Twitter
|
||||
With .ItemF({0, "content", "items", 0, "item", "itemContent", "tweet_results", "result", "tweet", "community_results", "result"})
|
||||
If .ListExists Then
|
||||
If ID = .Value("id_str") Then
|
||||
UserSiteNameUpdate(.Value("name"))
|
||||
UserSiteNameUpdate(.Value("name"), True)
|
||||
UserDescriptionUpdate(.Value("description"))
|
||||
|
||||
icon = .Value({"custom_banner_media", "media_info"}, "original_img_url").
|
||||
@@ -543,12 +543,12 @@ Namespace API.Twitter
|
||||
If .ListExists Then
|
||||
If ID.IsEmptyString Then ID = .Value("rest_id")
|
||||
icon = .Value({"avatar"}, "image_url")
|
||||
UserSiteNameUpdate(.Value({"core"}, "name"))
|
||||
UserSiteNameUpdate(.Value({"core"}, "name"), True)
|
||||
Dim tScreenName$ = .Value({"core"}, "screen_name")
|
||||
With .Item({"legacy"})
|
||||
If .ListExists Then
|
||||
If onlyUpdateUser Then
|
||||
If Not NameTrue = tScreenName Or 1 = 1 Then
|
||||
If Not NameTrue = tScreenName Then
|
||||
Dim uStr$ = $"username changed from '{NameTrue}' to '{tScreenName}'"
|
||||
LogError(Nothing, uStr)
|
||||
UserDescriptionUpdate(uStr, True, True, True)
|
||||
@@ -556,7 +556,7 @@ Namespace API.Twitter
|
||||
NameTrue = tScreenName
|
||||
End If
|
||||
If .Value("screen_name").IfNullOrEmpty(tScreenName).StringToLower = NameTrue.ToLower Then
|
||||
UserSiteNameUpdate(.Value("name"))
|
||||
UserSiteNameUpdate(.Value("name"), True)
|
||||
UserDescriptionUpdate(.Value("description"))
|
||||
|
||||
If icon.IsEmptyString Then icon = .Value("profile_image_url_https")
|
||||
@@ -1190,10 +1190,12 @@ nextpIndx:
|
||||
Dim cache As CacheKeeper = Nothing
|
||||
Try
|
||||
If ContentMissingExists Or (_ReparseLikes And LikesPosts.Count > 0) Then
|
||||
Const __entries$ = "entries"
|
||||
Dim m As UserMedia, mTmp As UserMedia
|
||||
Dim PostDate$
|
||||
Dim nodes As List(Of String()) = GetContainerSubnodes()
|
||||
Dim node$()
|
||||
Dim entriesNode As Predicate(Of EContainer) = Function(ee) ee.Contains(__entries)
|
||||
Dim j As EContainer, n As EContainer
|
||||
Dim f As SFile
|
||||
Dim i%, ii%
|
||||
@@ -1229,7 +1231,7 @@ nextpIndx:
|
||||
f = GDLRenameFile(files(ii), ii)
|
||||
j = JsonDocument.Parse(f.GetText)
|
||||
If Not j Is Nothing Then
|
||||
With j.ItemF({"data", 0, "instructions", 0, "entries"})
|
||||
With j.ItemF({"data", 0, "instructions", entriesNode, __entries})
|
||||
If .ListExists Then
|
||||
If IsSingleObjectDownload Or DownloadBroadcasts Then
|
||||
mTmp = ExtractBroadcast(.Self, m.Post.ID, String.Empty, nodes)
|
||||
|
||||
@@ -298,6 +298,7 @@ Namespace API.Xhamster
|
||||
containerNodes.Add({"videoListComponent", "videoThumbProps"})
|
||||
Else
|
||||
containerNodes.Add({"userGalleriesCollection"})
|
||||
containerNodes.Add({"contentComponent", "items"})
|
||||
End If
|
||||
End If
|
||||
|
||||
@@ -799,6 +800,9 @@ Namespace API.Xhamster
|
||||
Using ytdlp As New YTDLP.YTDLPBatch(TokenPersonal,, DestinationFile) : ytdlp.Encoding = Settings.CMDEncoding : ytdlp.Execute(cmd) : End Using
|
||||
Return DestinationFile
|
||||
End Function
|
||||
Protected Overrides Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
|
||||
Return DownloadContentDefault_ConvertWebp_TestImg(m, Process)
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Create media"
|
||||
Private Function ExtractMedia(ByVal j As EContainer, ByVal t As UTypes, Optional ByVal UrlNode As String = "pageURL",
|
||||
|
||||
@@ -23,7 +23,7 @@ Namespace API.Xhamster
|
||||
MyBase.New(s)
|
||||
GetMoments = s.GetMoments.Value
|
||||
End Sub
|
||||
Friend Overrides Sub Apply(ByRef u As IPSite)
|
||||
Friend Overrides Sub Apply(ByRef u As UserDataBase)
|
||||
MyBase.Apply(u)
|
||||
DirectCast(u, UserData).GetMoments = GetMoments
|
||||
End Sub
|
||||
|
||||
@@ -12,6 +12,7 @@ Imports PersonalUtilities.Forms
|
||||
Imports PersonalUtilities.Tools
|
||||
Imports SCrawler.API.Base
|
||||
Imports UserMediaD = SCrawler.DownloadObjects.TDownloader.UserMediaD
|
||||
Imports ImageRenderer2 = SCrawler.UserImage.ImageRenderer2
|
||||
Namespace DownloadObjects
|
||||
<ToolboxItem(False), DesignTimeVisible(False)>
|
||||
Public Class FeedMedia
|
||||
@@ -166,42 +167,6 @@ Namespace DownloadObjects
|
||||
Public Sub New()
|
||||
InitializeComponent()
|
||||
End Sub
|
||||
Private Class ImageRenderer2 : Inherits ImageRenderer
|
||||
Friend NativeFormat As String = Nothing
|
||||
Friend ImgErr As Exception = Nothing
|
||||
Friend Sub New(ByVal ImgPath As SFile, Optional ByVal e As ErrorsDescriber = Nothing)
|
||||
MyBase.New()
|
||||
Try
|
||||
If ImgPath.Exists(SFO.File, False) Then
|
||||
OriginalImageBytes = SFile.GetBytes(ImgPath, EDP.ThrowException)
|
||||
Try
|
||||
OriginalImage = GetImage(OriginalImageBytes)
|
||||
Catch exInternal As Exception
|
||||
HasError = True
|
||||
ImgErr = exInternal
|
||||
NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
|
||||
End Try
|
||||
End If
|
||||
Address = ImgPath
|
||||
Catch ex As Exception
|
||||
HasError = True
|
||||
NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
|
||||
If Not e.Exists Then e = EDP.ThrowException
|
||||
ErrorsDescriber.Execute(e, ex, $"ImageRenderer2.New({ImgPath})")
|
||||
End Try
|
||||
End Sub
|
||||
Friend Shared Function GetTrueFormat(ByVal Img() As Byte, Optional ByVal e As ErrorsDescriber = Nothing) As String
|
||||
Try
|
||||
Using ms As New MemoryStream(Img, 0, Img.Length)
|
||||
Return System.Windows.Media.Imaging.BitmapDecoder.Create(ms, Windows.Media.Imaging.BitmapCreateOptions.PreservePixelFormat,
|
||||
Windows.Media.Imaging.BitmapCacheOption.OnLoad).Metadata.Format
|
||||
End Using
|
||||
Catch ex As Exception
|
||||
If Not e.Exists Then e = EDP.ThrowException
|
||||
Return ErrorsDescriber.Execute(e, ex, "[ImageRenderer2.GetTrueFormat()]")
|
||||
End Try
|
||||
End Function
|
||||
End Class
|
||||
Friend Sub New(ByVal Media As UserMediaD, ByVal Width As Integer, ByVal Height As Integer, ByVal IsSession As Boolean, ByVal ExtractText As Boolean)
|
||||
Try
|
||||
InitializeComponent()
|
||||
|
||||
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
|
||||
' by using the '*' as shown below:
|
||||
' <Assembly: AssemblyVersion("1.0.*")>
|
||||
|
||||
<Assembly: AssemblyVersion("2026.2.14.0")>
|
||||
<Assembly: AssemblyFileVersion("2026.2.14.0")>
|
||||
<Assembly: AssemblyVersion("2026.3.23.0")>
|
||||
<Assembly: AssemblyFileVersion("2026.3.23.0")>
|
||||
<Assembly: NeutralResourcesLanguage("en")>
|
||||
|
||||
@@ -194,6 +194,7 @@
|
||||
<Compile Include="API\Base\TokenBatch.vb" />
|
||||
<Compile Include="API\Base\YTDLP.vb" />
|
||||
<Compile Include="API\Bluesky\Declarations.vb" />
|
||||
<Compile Include="API\Bluesky\EditorExchangeOptions.vb" />
|
||||
<Compile Include="API\Bluesky\M3U8.vb" />
|
||||
<Compile Include="API\Bluesky\SiteSettings.vb" />
|
||||
<Compile Include="API\Bluesky\UserData.vb" />
|
||||
|
||||
@@ -150,4 +150,45 @@ Friend Class UserImage : Inherits ImageRenderer
|
||||
Private Shared Function ConvertWebpTryImageMagick(ByVal InitFile As SFile, ByVal DestFile As SFile) As Boolean
|
||||
Return ImageRendererExt.ConvertWebp(InitFile, DestFile, EDP.SendToLog + EDP.ReturnValue)
|
||||
End Function
|
||||
Friend Class ImageRenderer2 : Inherits ImageRenderer
|
||||
Friend NativeFormat As String = Nothing
|
||||
Friend ImgErr As Exception = Nothing
|
||||
Friend ReadOnly Property IsWebP As Boolean
|
||||
Get
|
||||
Return NativeFormat.IfNullOrEmpty(ExtJpg).ToLower = ExtWebp
|
||||
End Get
|
||||
End Property
|
||||
Friend Sub New(ByVal ImgPath As SFile, Optional ByVal e As ErrorsDescriber = Nothing)
|
||||
MyBase.New()
|
||||
Try
|
||||
If ImgPath.Exists(SFO.File, False) Then
|
||||
OriginalImageBytes = SFile.GetBytes(ImgPath, EDP.ThrowException)
|
||||
Try
|
||||
OriginalImage = GetImage(OriginalImageBytes)
|
||||
Catch exInternal As Exception
|
||||
HasError = True
|
||||
ImgErr = exInternal
|
||||
NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
|
||||
End Try
|
||||
End If
|
||||
Address = ImgPath
|
||||
Catch ex As Exception
|
||||
HasError = True
|
||||
NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
|
||||
If Not e.Exists Then e = EDP.ThrowException
|
||||
ErrorsDescriber.Execute(e, ex, $"ImageRenderer2.New({ImgPath})")
|
||||
End Try
|
||||
End Sub
|
||||
Friend Shared Function GetTrueFormat(ByVal Img() As Byte, Optional ByVal e As ErrorsDescriber = Nothing) As String
|
||||
Try
|
||||
Using ms As New MemoryStream(Img, 0, Img.Length)
|
||||
Return System.Windows.Media.Imaging.BitmapDecoder.Create(ms, Windows.Media.Imaging.BitmapCreateOptions.PreservePixelFormat,
|
||||
Windows.Media.Imaging.BitmapCacheOption.OnLoad).Metadata.Format
|
||||
End Using
|
||||
Catch ex As Exception
|
||||
If Not e.Exists Then e = EDP.ThrowException
|
||||
Return ErrorsDescriber.Execute(e, ex, "[ImageRenderer2.GetTrueFormat()]")
|
||||
End Try
|
||||
End Function
|
||||
End Class
|
||||
End Class
|
||||
Reference in New Issue
Block a user