2025.3.17.0

API.SiteSettingsBase: fix incorrect class initializer
API.UserDataBase: add all objects to xml (STD)
API.Facebook: fix downloading reels from noname profiles
API.Pinterest: remove 'UserOptions' overrides (SiteSettings); add 'PwsHeader' to 'GetBoards'
API.PornHub: fix 'UpdateUserOptions' function ('NameTrue')
API.Threads: fix 'pinned' posts
API.TikTok: add photos download
This commit is contained in:
Andy
2025-03-17 16:23:41 +03:00
parent 2f838929cc
commit fff63d0a9f
13 changed files with 307 additions and 90 deletions

View File

@@ -1,3 +1,17 @@
# 2025.3.17.0
*2025-03-17*
- Added
- **TikTok: downloading photos**
- Updated
- gallery-dl up to version **1.29.2**
- Fixed
- Sites
- Facebook: reels aren't downloaded from noname profiles
- PornHub: newly added users aren't downloading
- Threads: users aren't updated if there is a pinned post
# 2025.2.25.0 # 2025.2.25.0
*2025-02-25* *2025-02-25*

View File

@@ -45,7 +45,7 @@ A program to download photo and video from [any site](#supported-sites) (e.g. Yo
- Instagram images and videos, tagged posts, stories, saved posts; - Instagram images and videos, tagged posts, stories, saved posts;
- Threads images and videos, saved posts; - Threads images and videos, saved posts;
- Facebook images and videos, stories, saved posts; - Facebook images and videos, stories, saved posts;
- TikTok videos; - TikTok images and videos;
- Pinterest boards, users, saved posts; - Pinterest boards, users, saved posts;
- Imgur images, galleries and videos; - Imgur images, galleries and videos;
- Gfycat videos; - Gfycat videos;

View File

@@ -394,7 +394,7 @@ Namespace API.Base
With c.GetParameters With c.GetParameters
If .ListExists Then If .ListExists Then
If .Count = 1 Then If .Count = 1 Then
Return .Self()(0).ParameterType.GetInterfaces.ListIfNothing.Where(Function(i) i Is Me.GetType).Count = 1 Return .Self()(0).ParameterType Is Me.GetType
Else Else
Return False Return False
End If End If
@@ -412,7 +412,8 @@ Namespace API.Base
End If End If
End With End With
If Not constructor Is Nothing Then If Not constructor Is Nothing Then
If args > 0 AndAlso Not constructor.GetParameters()(0).ParameterType Is GetType(ISiteSettings) Then Throw New Exception If args > 0 AndAlso constructor.GetParameters()(0).ParameterType.GetInterface(GetType(ISiteSettings).Name) Is Nothing Then _
Throw New Exception("Class Interface type is incompatible")
If args = 0 Then Options = constructor.Invoke(Nothing) Else Options = constructor.Invoke({Me}) If args = 0 Then Options = constructor.Invoke(Nothing) Else Options = constructor.Invoke({Me})
End If End If
If Options Is Nothing Then Options = Activator.CreateInstance(_UserOptionsType) If Options Is Nothing Then Options = Activator.CreateInstance(_UserOptionsType)

View File

@@ -1461,6 +1461,7 @@ BlockNullPicture:
Data.DownloadState = UserMediaStates.Missing Data.DownloadState = UserMediaStates.Missing
End If End If
YouTube.Objects.YouTubeMediaContainerBase.Update(_ContentNew(0), Data) YouTube.Objects.YouTubeMediaContainerBase.Update(_ContentNew(0), Data)
If _ContentNew.Count > 1 Then Data.Files.ListAddList(_ContentNew.Select(Function(cc) cc.File), LNC)
If ResetTitle And Not _ContentNew(0).File.Name.IsEmptyString Then Data.Title = _ContentNew(0).File.Name If ResetTitle And Not _ContentNew(0).File.Name.IsEmptyString Then Data.Title = _ContentNew(0).File.Name
Else Else
Data.DownloadState = UserMediaStates.Missing Data.DownloadState = UserMediaStates.Missing

View File

@@ -638,7 +638,7 @@ Namespace API.Facebook
End If End If
End Function End Function
Private Sub GetVideoPageID(ByVal GetReels As Boolean, ByVal Token As CancellationToken) Private Sub GetVideoPageID(ByVal GetReels As Boolean, ByVal Token As CancellationToken)
Dim URL$ = $"{GetProfileUrl()}\{IIf(GetReels, "reels", "videos")}" Dim URL$ = $"{GetProfileUrl()}{IIf(IsNoNameProfile, "&sk=", "/")}{IIf(GetReels, IIf(IsNoNameProfile, "reels_tab", "reels"), "videos")}"
Dim resp As Responser = HtmlResponserCreate() Dim resp As Responser = HtmlResponserCreate()
Try Try
WaitTimer() WaitTimer()

View File

@@ -31,6 +31,7 @@ Namespace API.Pinterest
CheckNetscapeCookiesOnEndInit = True CheckNetscapeCookiesOnEndInit = True
UseNetscapeCookies = True UseNetscapeCookies = True
UserRegex = RParams.DMS("https?://w{0,3}.?[^/]*?.?pinterest.com/([^/]+)/?(?(_)|([^/]*))/?([^/\?]*)", 0, RegexReturn.ListByMatch, EDP.ReturnValue) UserRegex = RParams.DMS("https?://w{0,3}.?[^/]*?.?pinterest.com/([^/]+)/?(?(_)|([^/]*))/?([^/\?]*)", 0, RegexReturn.ListByMatch, EDP.ReturnValue)
UserOptionsType = GetType(EditorExchangeOptions)
End Sub End Sub
#End Region #End Region
#Region "GetInstance, Available" #Region "GetInstance, Available"
@@ -72,12 +73,6 @@ Namespace API.Pinterest
Return String.Empty Return String.Empty
End If End If
End Function End Function
Friend Overrides Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean)
If Options Is Nothing Then Options = New EditorExchangeOptions
If OpenForm Then
Using f As New InternalSettingsForm(Options, Me, False) : f.ShowDialog() : End Using
End If
End Sub
#End Region #End Region
End Class End Class
End Namespace End Namespace

View File

@@ -170,6 +170,7 @@ Namespace API.Pinterest
urls.ListAddList(GetDataFromGalleryDL(URL, True, Token), LNC) urls.ListAddList(GetDataFromGalleryDL(URL, True, Token), LNC)
If urls.ListExists Then urls.RemoveAll(Function(__url) Not __url.Contains("BoardsResource/get/")) If urls.ListExists Then urls.RemoveAll(Function(__url) Not __url.Contains("BoardsResource/get/"))
If urls.ListExists Then If urls.ListExists Then
Responser.Headers.Add(PwsHeader)
ProgressPre.ChangeMax(urls.Count) ProgressPre.ChangeMax(urls.Count)
For Each URL In urls For Each URL In urls
ProgressPre.Perform() ProgressPre.Perform()
@@ -193,6 +194,8 @@ Namespace API.Pinterest
Catch ex As Exception Catch ex As Exception
ProcessException(ex, Token, $"data (gallery-dl boards) downloading error [{URL}]") ProcessException(ex, Token, $"data (gallery-dl boards) downloading error [{URL}]")
Return Nothing Return Nothing
Finally
Responser.Headers.Remove(PwsHeader)
End Try End Try
End Function End Function
Private Sub DownloadBoardImages(ByRef Board As BoardInfo, ByVal Token As CancellationToken) Private Sub DownloadBoardImages(ByRef Board As BoardInfo, ByVal Token As CancellationToken)

View File

@@ -195,7 +195,7 @@ Namespace API.PornHub
If Not Force OrElse (Not IsUser AndAlso Not SiteMode = SiteModes.Playlists AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then If Not Force OrElse (Not IsUser AndAlso Not SiteMode = SiteModes.Playlists AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then
Dim eObj As Plugin.ExchangeOptions = Nothing Dim eObj As Plugin.ExchangeOptions = Nothing
If Force Then eObj = MySettings.IsMyUser(NewUrl) If Force Then eObj = MySettings.IsMyUser(NewUrl)
If (Force And Not eObj.UserName.IsEmptyString) Or (Not Force And Not Name.IsEmptyString And NameTrue.IsEmptyString) Then If (Force And Not eObj.UserName.IsEmptyString) Or (Not Force And Not Name.IsEmptyString And NameTrue(True).IsEmptyString) Then
If Not If(Force, eObj.Options, Options).IsEmptyString Then If Not If(Force, eObj.Options, Options).IsEmptyString Then
If (IsUser Or SiteMode = SiteModes.Playlists) And Force Then If (IsUser Or SiteMode = SiteModes.Playlists) And Force Then
Return False Return False
@@ -241,7 +241,7 @@ Namespace API.PornHub
SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User) SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
UpdateUserOptions() UpdateUserOptions()
Else Else
If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString : .Value(Name_TrueName) = NameTrue(True)
.Add(Name_PersonType, PersonType) .Add(Name_PersonType, PersonType)
.Add(Name_DownloadUHD, DownloadUHD.BoolToInteger) .Add(Name_DownloadUHD, DownloadUHD.BoolToInteger)
.Add(Name_DownloadUploaded, DownloadUploaded.BoolToInteger) .Add(Name_DownloadUploaded, DownloadUploaded.BoolToInteger)

View File

@@ -128,10 +128,13 @@ Namespace API.ThreadsNet
If IsSavedPosts Then If IsSavedPosts Then
Return False Return False
Else Else
With Items(Index).ItemF(DefaultParser_ElemNode)
Return .Value({"text_post_app_info", "pinned_post_info"}, "is_pinned_to_profile").FromXML(Of Boolean)(False)
If MaxLastDownDate.HasValue Then If MaxLastDownDate.HasValue Then
Dim d As Date? = AConvert(Of Date)(Items(Index).ItemF(DefaultParser_ElemNode_Default).Value("taken_at"), UnixDate32Provider, Nothing) Dim d As Date? = AConvert(Of Date)(.Value("taken_at"), UnixDate32Provider, Nothing)
If d.HasValue Then Return d.Value < MaxLastDownDate.Value If d.HasValue Then Return d.Value <= MaxLastDownDate.Value
End If End If
End With
Return Not FirstLoadingDone Return Not FirstLoadingDone
End If End If
Catch ex As Exception Catch ex As Exception

View File

@@ -6,11 +6,14 @@
' '
' This program is distributed in the hope that it will be useful, ' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY ' but WITHOUT ANY WARRANTY
Imports System.Text.RegularExpressions
Imports PersonalUtilities.Functions.RegularExpressions Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.TikTok Namespace API.TikTok
Friend Module Declarations Friend Module Declarations
Friend ReadOnly SimpleDateConverter As New ADateTime("yyyyMMdd") Friend ReadOnly SimpleDateConverter As New ADateTime("yyyyMMdd")
Friend ReadOnly RegexTagsReplacer As RParams = RParams.DM("#\w+\s?", -1, RegexReturn.Replace, Friend ReadOnly RegexTagsReplacer As RParams = RParams.DM("#\w+\s?", -1, RegexReturn.Replace,
CType(Function(input$) String.Empty, Func(Of String, String)), EDP.ReturnValue) CType(Function(input$) String.Empty, Func(Of String, String)), EDP.ReturnValue)
Friend ReadOnly RegexPhotoJson As RParams = RParams.DMS("UNIVERSAL_DATA_FOR_REHYDRATION__"" type=""application/json""\>([^\<]+)\<", 1,
RegexOptions.IgnoreCase, EDP.ReturnValue)
End Module End Module
End Namespace End Namespace

View File

@@ -13,6 +13,15 @@ Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.TikTok Namespace API.TikTok
<Manifest("AndyProgram_TikTok"), SpecialForm(False), SeparatedTasks(1)> <Manifest("AndyProgram_TikTok"), SpecialForm(False), SeparatedTasks(1)>
Friend Class SiteSettings : Inherits SiteSettingsBase Friend Class SiteSettings : Inherits SiteSettingsBase
#Region "Categories"
Private Const CAT_DOWN As String = "Download"
#End Region
#Region "Download"
<PropertyOption(ControlText:="Download videos", Category:=CAT_DOWN), PXML, PClonable>
Friend ReadOnly Property DownloadTTVideos As PropertyValue
<PropertyOption(ControlText:="Download photos", Category:=CAT_DOWN), PXML, PClonable>
Friend ReadOnly Property DownloadTTPhotos As PropertyValue
#End Region
<PropertyOption(ControlText:="Remove tags from title"), PXML, PClonable> <PropertyOption(ControlText:="Remove tags from title"), PXML, PClonable>
Friend ReadOnly Property RemoveTagsFromTitle As PropertyValue Friend ReadOnly Property RemoveTagsFromTitle As PropertyValue
<PropertyOption(ControlText:="Use native title", ControlToolTip:="Use a user-created video title for the filename instead of the video ID."), PXML, PClonable> <PropertyOption(ControlText:="Use native title", ControlToolTip:="Use a user-created video title for the filename instead of the video ID."), PXML, PClonable>
@@ -36,6 +45,10 @@ Namespace API.TikTok
Friend ReadOnly Property UseParsedVideoDateSTD As PropertyValue Friend ReadOnly Property UseParsedVideoDateSTD As PropertyValue
Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean) Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
MyBase.New("TikTok", "www.tiktok.com", AccName, Temp, My.Resources.SiteResources.TikTokIcon_32, My.Resources.SiteResources.TikTokPic_192) MyBase.New("TikTok", "www.tiktok.com", AccName, Temp, My.Resources.SiteResources.TikTokIcon_32, My.Resources.SiteResources.TikTokPic_192)
DownloadTTVideos = New PropertyValue(True)
DownloadTTPhotos = New PropertyValue(True)
RemoveTagsFromTitle = New PropertyValue(False) RemoveTagsFromTitle = New PropertyValue(False)
TitleUseNative = New PropertyValue(True) TitleUseNative = New PropertyValue(True)
TitleUseNativeSTD = New PropertyValue(True) TitleUseNativeSTD = New PropertyValue(True)
@@ -45,6 +58,7 @@ Namespace API.TikTok
TitleUseRegexForTitle_Value = New PropertyValue(String.Empty, GetType(String)) TitleUseRegexForTitle_Value = New PropertyValue(String.Empty, GetType(String))
UseParsedVideoDate = New PropertyValue(True) UseParsedVideoDate = New PropertyValue(True)
UseParsedVideoDateSTD = New PropertyValue(False) UseParsedVideoDateSTD = New PropertyValue(False)
UseNetscapeCookies = True UseNetscapeCookies = True
UrlPatternUser = "https://www.tiktok.com/@{0}/" UrlPatternUser = "https://www.tiktok.com/@{0}/"
UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "tiktok.com/@"), 1) UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "tiktok.com/@"), 1)

View File

@@ -13,6 +13,7 @@ Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools Imports PersonalUtilities.Tools
Imports PersonalUtilities.Tools.Web.Documents.JSON Imports PersonalUtilities.Tools.Web.Documents.JSON
Imports UTypes = SCrawler.API.Base.UserMedia.Types
Namespace API.TikTok Namespace API.TikTok
Friend Class UserData : Inherits UserDataBase Friend Class UserData : Inherits UserDataBase
#Region "XML names" #Region "XML names"
@@ -23,6 +24,7 @@ Namespace API.TikTok
Private Const Name_TitleUseRegexForTitle As String = "TitleUseRegexForTitle" Private Const Name_TitleUseRegexForTitle As String = "TitleUseRegexForTitle"
Private Const Name_TitleUseRegexForTitle_Value As String = "TitleUseRegexForTitle_Value" Private Const Name_TitleUseRegexForTitle_Value As String = "TitleUseRegexForTitle_Value"
Private Const Name_TitleUseGlobalRegexOptions As String = "TitleUseGlobalRegexOptions" Private Const Name_TitleUseGlobalRegexOptions As String = "TitleUseGlobalRegexOptions"
Private Const Name_PhotosDownloaded As String = "PhotosDownloaded"
#End Region #End Region
#Region "Declarations" #Region "Declarations"
Private ReadOnly Property MySettings As SiteSettings Private ReadOnly Property MySettings As SiteSettings
@@ -62,6 +64,7 @@ Namespace API.TikTok
Friend Property TitleUseRegexForTitle_Value As String = String.Empty Friend Property TitleUseRegexForTitle_Value As String = String.Empty
Friend Property TitleUseGlobalRegexOptions As Boolean = True Friend Property TitleUseGlobalRegexOptions As Boolean = True
Private Property LastDownloadDate As Date? = Nothing Private Property LastDownloadDate As Date? = Nothing
Private Property PhotosDownloaded As Boolean = False
#End Region #End Region
#Region "Exchange" #Region "Exchange"
Friend Overrides Function ExchangeOptionsGet() As Object Friend Overrides Function ExchangeOptionsGet() As Object
@@ -92,6 +95,7 @@ Namespace API.TikTok
TitleUseRegexForTitle = .Value(Name_TitleUseRegexForTitle).FromXML(Of Boolean)(False) TitleUseRegexForTitle = .Value(Name_TitleUseRegexForTitle).FromXML(Of Boolean)(False)
TitleUseRegexForTitle_Value = .Value(Name_TitleUseRegexForTitle_Value) TitleUseRegexForTitle_Value = .Value(Name_TitleUseRegexForTitle_Value)
TitleUseGlobalRegexOptions = .Value(Name_TitleUseGlobalRegexOptions).FromXML(Of Boolean)(True) TitleUseGlobalRegexOptions = .Value(Name_TitleUseGlobalRegexOptions).FromXML(Of Boolean)(True)
PhotosDownloaded = .Value(Name_PhotosDownloaded).FromXML(Of Boolean)(False)
Else Else
.Add(Name_RemoveTagsFromTitle, RemoveTagsFromTitle.BoolToInteger) .Add(Name_RemoveTagsFromTitle, RemoveTagsFromTitle.BoolToInteger)
.Add(Name_TitleUseNative, TitleUseNative.BoolToInteger) .Add(Name_TitleUseNative, TitleUseNative.BoolToInteger)
@@ -100,6 +104,7 @@ Namespace API.TikTok
.Add(Name_TitleUseRegexForTitle, TitleUseRegexForTitle.BoolToInteger) .Add(Name_TitleUseRegexForTitle, TitleUseRegexForTitle.BoolToInteger)
.Add(Name_TitleUseRegexForTitle_Value, TitleUseRegexForTitle_Value) .Add(Name_TitleUseRegexForTitle_Value, TitleUseRegexForTitle_Value)
.Add(Name_TitleUseGlobalRegexOptions, TitleUseGlobalRegexOptions.BoolToInteger) .Add(Name_TitleUseGlobalRegexOptions, TitleUseGlobalRegexOptions.BoolToInteger)
.Add(Name_PhotosDownloaded, PhotosDownloaded.BoolToInteger)
End If End If
End With End With
End Sub End Sub
@@ -142,7 +147,7 @@ Namespace API.TikTok
End Function End Function
Private Function GetNewFileName(ByVal Title As String, ByVal Native As Boolean, ByVal RemoveTags As Boolean, ByVal AddVideoID As Boolean, Private Function GetNewFileName(ByVal Title As String, ByVal Native As Boolean, ByVal RemoveTags As Boolean, ByVal AddVideoID As Boolean,
ByVal PostID As String, ByVal TitleRegex As RParams) As String ByVal PostID As String, ByVal TitleRegex As RParams) As String
If Not Title.IsEmptyString Then Title = Left(Title, 150).StringTrim If Not Title.IsEmptyString Then Title = TitleHtmlConverter(Left(Title, 150)).StringTrim
If Title.IsEmptyString Or Not Native Then If Title.IsEmptyString Or Not Native Then
Title = PostID Title = PostID
Else Else
@@ -157,6 +162,9 @@ Namespace API.TikTok
End If End If
Return Title Return Title
End Function End Function
Private Function GetPhotoNode() As Object()
Return {"imageURL", "urlList", 0, 0}
End Function
Friend Overrides Sub DownloadData(ByVal Token As CancellationToken) Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
MyBase.DownloadData(Token) MyBase.DownloadData(Token)
UserCache.DisposeIfReady(False) UserCache.DisposeIfReady(False)
@@ -166,13 +174,20 @@ Namespace API.TikTok
Dim URL$ = $"https://www.tiktok.com/@{NameTrue}" Dim URL$ = $"https://www.tiktok.com/@{NameTrue}"
UserCache = CreateCache() UserCache = CreateCache()
Try Try
Dim postID$, title$, postUrl$, newName$ Const photoPrefix$ = "photo_"
Dim postID$, title$, postUrl$, newName$, t$, postID2$, imgUrl$
Dim postDate As Date? Dim postDate As Date?
Dim dateAfterC As Date? = Nothing Dim dateAfterC As Date? = Nothing
Dim dateBefore As Date? = DownloadDateTo Dim dateBefore As Date? = DownloadDateTo
Dim dateAfter As Date? = DownloadDateFrom Dim dateAfter As Date? = DownloadDateFrom
Dim baseDataObtained As Boolean = False Dim baseDataObtained As Boolean = False
Dim titleRegex As RParams = GetTitleRegex() Dim titleRegex As RParams = GetTitleRegex()
Dim vPath As SFile = Nothing, pPath As SFile = Nothing
Dim file As SFile
Dim j As EContainer, photo As EContainer
Dim photoNode As Object() = GetPhotoNode()
Dim c%, cc%, i%
Dim errDef As New ErrorsDescriber(EDP.ReturnValue)
If _ContentList.Count > 0 Then If _ContentList.Count > 0 Then
With (From d In _ContentList Where d.Post.Date.HasValue Select d.Post.Date.Value) With (From d In _ContentList Where d.Post.Date.HasValue Select d.Post.Date.Value)
@@ -198,20 +213,44 @@ Namespace API.TikTok
End If End If
End If End If
If DownloadVideos And Settings.YtdlpFile.Exists And CBool(MySettings.DownloadTTVideos.Value) Then
With UserCache.NewInstance : .Validate() : vPath = .RootDirectory : End With
Using b As New YTDLP.YTDLPBatch(Token) With {.TempPostsList = _TempPostsList} Using b As New YTDLP.YTDLPBatch(Token) With {.TempPostsList = _TempPostsList}
b.Commands.Clear() b.Commands.Clear()
b.ChangeDirectory(UserCache) b.ChangeDirectory(vPath)
b.Encoding = BatchExecutor.UnicodeEncoding b.Encoding = BatchExecutor.UnicodeEncoding
b.Execute(CreateYTCommand(UserCache.RootDirectory, URL, False, dateBefore, dateAfter)) b.Execute(CreateYTCommand(vPath, URL, False, dateBefore, dateAfter))
End Using End Using
End If
If DownloadImages And Settings.GalleryDLFile.Exists And CBool(MySettings.DownloadTTPhotos.Value) Then
With UserCache.NewInstance : .Validate() : pPath = .RootDirectory : End With
Using b As New GDL.GDLBatch(Token)
With b
If PhotosDownloaded And _TempPostsList.Count > 0 Then
.TempPostsList = (From p As String In _TempPostsList
Where Not p.IsEmptyString AndAlso p.StartsWith(photoPrefix)
Select p.Replace(photoPrefix, String.Empty)).ListIfNothing
Else
.TempPostsList = New List(Of String)
End If
.ChangeDirectory(pPath)
.Encoding = BatchExecutor.UnicodeEncoding
.Execute(CreateGDLCommand(URL))
If Not PhotosDownloaded Then _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
PhotosDownloaded = True
End With
End Using
End If
ThrowAny(Token) ThrowAny(Token)
Dim files As List(Of SFile) = SFile.GetFiles(UserCache, "*.json",, EDP.ReturnValue) Dim files As List(Of SFile)
If Not vPath.IsEmptyString AndAlso vPath.Exists(SFO.Path, False) Then
files = SFile.GetFiles(vPath, "*.json",, errDef)
If files.ListExists Then If files.ListExists Then
Dim j As EContainer For Each file In files
For Each file As SFile In files j = JsonDocument.Parse(file.GetText, errDef)
j = JsonDocument.Parse(file.GetText, EDP.ReturnValue)
If j.ListExists Then If j.ListExists Then
If j.Value("_type").StringToLower = "video" Then If j.Value("_type").StringToLower = "video" Then
If Not baseDataObtained Then If Not baseDataObtained Then
@@ -227,9 +266,9 @@ Namespace API.TikTok
End If End If
postID = j.Value("id") postID = j.Value("id")
If Not _TempPostsList.Contains(postID) Then If Not _TempPostsList.Contains(postID) Then
_TempPostsList.Add(postID) _TempPostsList.ListAddValue(postID, LNC)
Else Else
Exit Sub Exit For 'Exit Sub
End If End If
title = GetNewFileName(j.Value("title").StringRemoveWinForbiddenSymbols, title = GetNewFileName(j.Value("title").StringRemoveWinForbiddenSymbols,
TitleUseNative, RemoveTagsFromTitle, TitleAddVideoID, postID, titleRegex) TitleUseNative, RemoveTagsFromTitle, TitleAddVideoID, postID, titleRegex)
@@ -237,18 +276,68 @@ Namespace API.TikTok
If Not postDate.HasValue Then postDate = AConvert(Of Date)(j.Value("upload_date"), SimpleDateConverter, Nothing) If Not postDate.HasValue Then postDate = AConvert(Of Date)(j.Value("upload_date"), SimpleDateConverter, Nothing)
Select Case CheckDatesLimit(postDate, SimpleDateConverter) Select Case CheckDatesLimit(postDate, SimpleDateConverter)
Case DateResult.Skip : Continue For Case DateResult.Skip : Continue For
Case DateResult.Exit : Exit Sub Case DateResult.Exit : Exit For 'Exit Sub
End Select End Select
postUrl = j.Value("webpage_url") postUrl = j.Value("webpage_url")
If postUrl.IsEmptyString Then postUrl = $"https://www.tiktok.com/@{Name}/video/{postID}" If postUrl.IsEmptyString Then postUrl = $"https://www.tiktok.com/@{Name}/video/{postID}"
_TempMediaList.Add(New UserMedia(postUrl, UserMedia.Types.Video) With { _TempMediaList.Add(New UserMedia(postUrl, UTypes.Video) With {
.File = $"{title}.mp4", .Post = New UserPost(postID, postDate)}) .File = $"{title}.mp4", .Post = New UserPost(postID, postDate)})
End If End If
j.Dispose() j.Dispose()
End If End If
Next Next
End If End If
End If
If Not pPath.IsEmptyString AndAlso pPath.Exists(SFO.Path, False) Then
files = SFile.GetFiles(pPath, "*.txt",, errDef)
If files.ListExists Then
For Each file In files
t = file.GetText(errDef)
If Not t.IsEmptyString Then t = RegexReplace(t, RegexPhotoJson)
If Not t.IsEmptyString Then
j = JsonDocument.Parse(t, errDef)
If j.ListExists Then
With j.ItemF({0, "webapp.video-detail", "itemInfo", "itemStruct"})
If .ListExists Then
postID = .Value("id")
postID2 = $"{photoPrefix}{postID}"
If Not _TempPostsList.Contains(postID2) Then _TempPostsList.ListAddValue(postID2, LNC) Else Exit For 'Exit Sub
postDate = AConvert(Of Date)(j.Value("createTime"), UnixDate32Provider, Nothing)
Select Case CheckDatesLimit(postDate, SimpleDateConverter)
Case DateResult.Skip : Continue For
Case DateResult.Exit : Exit For 'Exit Sub
End Select
title = GetNewFileName(j.Value({"imagePost"}, "title").StringRemoveWinForbiddenSymbols,
TitleUseNative, RemoveTagsFromTitle, TitleAddVideoID, postID, titleRegex)
postUrl = $"https://www.tiktok.com/@{Name}/photo/{postID}"
With .Item({"imagePost", "images"})
If .ListExists Then
i = 0
c = .Count
cc = Math.Max(c.ToString.Length, 3)
For Each photo In .Self
i += 1
imgUrl = photo.ItemF(photoNode).XmlIfNothingValue
If Not imgUrl.IsEmptyString Then _
_TempMediaList.Add(New UserMedia(imgUrl, UTypes.Picture) With {
.URL_BASE = postUrl,
.SpecialFolder = "Photo",
.File = $"{title}{IIf(c > 1, $"_{i.NumToString(ANumbers.Formats.NumberGroup, cc)}", String.Empty)}.jpg",
.Post = New UserPost(postID, postDate)})
Next
End If
End With
End If
End With
j.Dispose()
End If
End If
Next
End If
End If
If _TempMediaList.Count > 0 Then LastDownloadDate = Now If _TempMediaList.Count > 0 Then LastDownloadDate = Now
Catch ex As Exception Catch ex As Exception
ProcessException(ex, Token, $"data downloading error [{URL}]") ProcessException(ex, Token, $"data downloading error [{URL}]")
@@ -259,16 +348,41 @@ Namespace API.TikTok
Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken) Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
If ContentMissingExists Then If ContentMissingExists Then
Dim m As UserMedia Dim m As UserMedia
Dim d As IYouTubeMediaContainer = Nothing
Dim i% Dim i%
Dim rList As New List(Of Integer) Dim rList As New List(Of Integer)
Dim picIDs As New List(Of String)
Dim defDir As SFile = SFile.GetPath(DownloadContentDefault_GetRootDir())
Dim result As Boolean
For i = 0 To _ContentList.Count - 1 For i = 0 To _ContentList.Count - 1
If _ContentList(i).State = UserMedia.States.Missing Then If _ContentList(i).State = UserMedia.States.Missing Then
m = _ContentList(i) m = _ContentList(i)
m.URL = m.URL_BASE result = False
_TempMediaList.Add(m) Try
If m.Type = UTypes.Video Then
d = MySettings.GetSingleMediaInstance(m.URL_BASE, defDir)
result = False
If If(UserCache?.Disposed, True) Then UserCache = CreateCache()
DownloadSingleObject_GetPosts(d, Token, UserCache, result)
ElseIf m.Type = UTypes.Picture Then
If picIDs.Contains(m.Post.ID) Then
rList.Add(i) rList.Add(i)
Else
d = MySettings.GetSingleMediaInstance(m.URL_BASE, defDir)
If If(UserCache?.Disposed, True) Then UserCache = CreateCache()
DownloadSingleObject_GetPosts(d, Token, UserCache, result)
picIDs.Add(m.Post.ID)
End If
End If
Catch ex As Exception
result = False
ProcessException(ex, Token, "ReparseMissing")
End Try
If result Then rList.Add(i)
d.DisposeIfReady(False)
End If End If
Next Next
picIDs.Clear()
If rList.Count > 0 Then If rList.Count > 0 Then
For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(rList(i)) : Next For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(rList(i)) : Next
End If End If
@@ -303,10 +417,18 @@ Namespace API.TikTok
Return command Return command
End Function End Function
#End Region #End Region
#Region "GDL Support"
Private Function CreateGDLCommand(ByVal URL As String) As String
Return $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --write-pages {URL}"
End Function
#End Region
#Region "DownloadContent, DownloadFile" #Region "DownloadContent, DownloadFile"
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken) Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
DownloadContentDefault(Token) DownloadContentDefault(Token)
End Sub End Sub
Protected Overrides Function ValidateDownloadFile(ByVal URL As String, ByVal Media As UserMedia, ByRef Interrupt As Boolean) As Boolean
Return Not Media.Type = UTypes.Picture
End Function
Protected Overrides Function DownloadFile(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile Protected Overrides Function DownloadFile(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile
Using b As New TokenBatch(Token) With {.FileExchanger = RootCacheTikTok} Using b As New TokenBatch(Token) With {.FileExchanger = RootCacheTikTok}
b.Encoding = BatchExecutor.UnicodeEncoding b.Encoding = BatchExecutor.UnicodeEncoding
@@ -316,8 +438,18 @@ Namespace API.TikTok
End Function End Function
#End Region #End Region
#Region "DownloadSingleObject" #Region "DownloadSingleObject"
Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken) Protected Overloads Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
DownloadSingleObject_GetPosts(Data, Token, Nothing, Nothing)
End Sub
Private Overloads Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken,
ByRef Cache As CacheKeeper, ByRef Result As Boolean)
Dim f$ = String.Empty Dim f$ = String.Empty
Dim urlsList As New List(Of String)
Dim t As UTypes
Dim defName$ = New SFile(Data.URL).Name
If Data.URL.ToLower.Contains("/video/") Then
urlsList.Add(Data.URL)
t = UTypes.Video
If CBool(MySettings.TitleUseNativeSTD.Value) Then If CBool(MySettings.TitleUseNativeSTD.Value) Then
Using b As New BatchExecutor(True) With { Using b As New BatchExecutor(True) With {
.Encoding = BatchExecutor.UnicodeEncoding, .Encoding = BatchExecutor.UnicodeEncoding,
@@ -335,14 +467,65 @@ Namespace API.TikTok
End With End With
End Using End Using
End If End If
Dim m As New UserMedia(Data.URL, UserMedia.Types.Video) Else
If Not f.IsEmptyString Then f = TitleHtmlConverter(f) t = UTypes.Picture
If Not f.IsEmptyString Then Data.ContentType = Plugin.UserMediaTypes.Picture
f = GetNewFileName(f, MySettings.TitleUseNativeSTD.Value, MySettings.RemoveTagsFromTitle.Value, MySettings.TitleAddVideoIDSTD.Value, Data.Title = defName
m.File.Name, GetTitleRegex) Dim dir As SFile
If Not f.IsEmptyString Then m.File.Name = f.StringTrim With If(Cache, Settings.Cache).NewInstance() : .Validate() : dir = .RootDirectory : End With
Using b As New GDL.GDLBatch(Token)
b.ChangeDirectory(dir)
b.Encoding = BatchExecutor.UnicodeEncoding
b.Execute(CreateGDLCommand(Data.URL))
End Using
Dim file As SFile = SFile.GetFiles(dir, "*.txt",, EDP.ReturnValue).FirstOrDefault
If file.Exists Then
Dim r$ = file.GetText(EDP.ReturnValue)
If Not r.IsEmptyString Then r = RegexReplace(r, RegexPhotoJson)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
If j.ListExists Then
With j.ItemF({0, "webapp.video-detail", "itemInfo", "itemStruct"})
If CBool(MySettings.TitleUseNativeSTD.Value) Then f = j.Value({"imagePost"}, "title").StringRemoveWinForbiddenSymbols
With .Item({"imagePost", "images"})
If .ListExists Then
For Each photo As EContainer In .Self : urlsList.Add(photo.ItemF(GetPhotoNode()).XmlIfNothingValue) : Next
End If End If
End With
End With
End If
End Using
End If
End If
End If
Dim m As UserMedia
Dim i% = 0, c%, cc%
Dim ff As Boolean = False
If urlsList.Count > 0 Then
c = urlsList.Count
cc = Math.Max(c.ToString.Length, 3)
For Each url$ In urlsList
i += 1
m = New UserMedia(url, t) With {.URL_BASE = Data.URL}
If Not f.IsEmptyString Then f = TitleHtmlConverter(f)
If Not f.IsEmptyString Or t = UTypes.Picture Then
If Not ff Then f = GetNewFileName(f, MySettings.TitleUseNativeSTD.Value, MySettings.RemoveTagsFromTitle.Value, MySettings.TitleAddVideoIDSTD.Value,
defName, GetTitleRegex)
ff = True
If Not f.IsEmptyString Then
m.File.Name = $"{f.StringTrim}{IIf(c > 1, $"_{i.NumToString(ANumbers.Formats.NumberGroup, cc)}", String.Empty)}"
If t = UTypes.Picture Then m.File.Extension = "jpg"
End If
End If
_TempMediaList.Add(m) _TempMediaList.Add(m)
Result = True
Next
End If
End Sub
Protected Overrides Sub DownloadSingleObject_PostProcessing(ByVal Data As IYouTubeMediaContainer, Optional ByVal ResetTitle As Boolean = True)
MyBase.DownloadSingleObject_PostProcessing(Data, Not Data.ContentType = Plugin.UserMediaTypes.Picture)
End Sub End Sub
#End Region #End Region
#Region "EraseData" #Region "EraseData"

View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below: ' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")> ' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2025.2.25.0")> <Assembly: AssemblyVersion("2025.3.17.0")>
<Assembly: AssemblyFileVersion("2025.2.25.0")> <Assembly: AssemblyFileVersion("2025.3.17.0")>
<Assembly: NeutralResourcesLanguage("en")> <Assembly: NeutralResourcesLanguage("en")>