mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 00:02:17 +00:00
2.0.0.0
Added Instagram downloading, filter by site, channels groups, change folder function, imgur compatibility, special folders, deleting with keeping data, Reddit saved posts downloading Fixed limited twitter downloading, suspended profiles Updated download algo Concat sites editors into a single form Updated Reddit downloading algo Fixed saved function in video downloader Some improvements
This commit is contained in:
Andy
13
SCrawler/API/Base/Declarations.vb
Normal file
13
SCrawler/API/Base/Declarations.vb
Normal file
@@ -0,0 +1,13 @@
|
||||
' Copyright (C) 2022 Andy
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Namespace API.Base
|
||||
Friend Module Declarations
|
||||
Friend ReadOnly LNC As New ListAddParams(LAP.NotContainsOnly)
|
||||
End Module
|
||||
End Namespace
|
||||
@@ -25,8 +25,47 @@ Namespace API.Base
|
||||
_Path.Value = NewFile
|
||||
End Set
|
||||
End Property
|
||||
Friend ReadOnly Property InstaHash As XMLValue(Of String)
|
||||
Friend ReadOnly Property InstaHashUpdateRequired As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property InstagramDownloadingErrorDate As XMLValue(Of Date)
|
||||
Friend Property InstagramLastApplyingValue As Integer? = Nothing
|
||||
Friend ReadOnly Property InstagramReadyForDownload As Boolean
|
||||
Get
|
||||
With InstagramDownloadingErrorDate
|
||||
If .ValueF.Exists Then
|
||||
Return .ValueF.Value.AddMinutes(If(InstagramLastApplyingValue, 10)) < Now
|
||||
Else
|
||||
Return True
|
||||
End If
|
||||
End With
|
||||
End Get
|
||||
End Property
|
||||
Friend Property InstagramTooManyRequestsReadyForCatch As Boolean = True
|
||||
Friend Sub InstagramTooManyRequests(ByVal Catched As Boolean)
|
||||
With InstagramDownloadingErrorDate
|
||||
If Catched Then
|
||||
If Not .ValueF.Exists Then
|
||||
.Value = Now
|
||||
If InstagramTooManyRequestsReadyForCatch Then
|
||||
InstagramLastApplyingValue = If(InstagramLastApplyingValue, 0) + 10
|
||||
InstagramTooManyRequestsReadyForCatch = False
|
||||
MyMainLOG = $"Instagram downloading error: too many requests. Try again after {If(InstagramLastApplyingValue, 10)} minutes..."
|
||||
End If
|
||||
End If
|
||||
Else
|
||||
.ValueF = Nothing
|
||||
InstagramLastApplyingValue = Nothing
|
||||
End If
|
||||
End With
|
||||
End Sub
|
||||
Friend ReadOnly Property Temporary As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property DownloadImages As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property DownloadVideos As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property GetUserMediaOnly As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property SavedPostsUserName As XMLValue(Of String)
|
||||
Private ReadOnly SettingsFile As SFile
|
||||
Friend Sub New(ByVal s As Sites, ByRef _XML As XmlFile, ByVal GlobalPath As SFile)
|
||||
Friend Sub New(ByVal s As Sites, ByRef _XML As XmlFile, ByVal GlobalPath As SFile,
|
||||
ByRef _Temp As XMLValue(Of Boolean), ByRef _Imgs As XMLValue(Of Boolean), ByRef _Vids As XMLValue(Of Boolean))
|
||||
Site = s
|
||||
SettingsFile = $"{SettingsFolderName}\Responser_{s}.xml"
|
||||
Responser = New WEB.Response(SettingsFile)
|
||||
@@ -34,40 +73,112 @@ Namespace API.Base
|
||||
If SettingsFile.Exists Then
|
||||
Responser.LoadSettings()
|
||||
Else
|
||||
If Site = Sites.Twitter Then
|
||||
With Responser
|
||||
.ContentType = "application/json"
|
||||
.Accept = "*/*"
|
||||
.CookiesDomain = "twitter.com"
|
||||
.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
With .Headers
|
||||
.Add("sec-ch-ua", " Not;A Brand" & Chr(34) & ";v=" & Chr(34) & "99" & Chr(34) & ", " & Chr(34) &
|
||||
"Google Chrome" & Chr(34) & ";v=" & Chr(34) & "91" & Chr(34) & ", " & Chr(34) & "Chromium" &
|
||||
Chr(34) & ";v=" & Chr(34) & "91" & Chr(34))
|
||||
.Add("sec-ch-ua-mobile", "?0")
|
||||
.Add("sec-fetch-dest", "empty")
|
||||
.Add("sec-fetch-mode", "cors")
|
||||
.Add("sec-fetch-site", "same-origin")
|
||||
.Add(Header_Twitter_Token, String.Empty)
|
||||
.Add("x-twitter-active-user", "yes")
|
||||
.Add("x-twitter-auth-type", "OAuth2Session")
|
||||
.Add(Header_Twitter_Authorization, String.Empty)
|
||||
Select Case Site
|
||||
Case Sites.Twitter
|
||||
With Responser
|
||||
.ContentType = "application/json"
|
||||
.Accept = "*/*"
|
||||
.CookiesDomain = "twitter.com"
|
||||
.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
With .Headers
|
||||
.Add("sec-ch-ua", " Not;A Brand" & Chr(34) & ";v=" & Chr(34) & "99" & Chr(34) & ", " & Chr(34) &
|
||||
"Google Chrome" & Chr(34) & ";v=" & Chr(34) & "91" & Chr(34) & ", " & Chr(34) & "Chromium" &
|
||||
Chr(34) & ";v=" & Chr(34) & "91" & Chr(34))
|
||||
.Add("sec-ch-ua-mobile", "?0")
|
||||
.Add("sec-fetch-dest", "empty")
|
||||
.Add("sec-fetch-mode", "cors")
|
||||
.Add("sec-fetch-site", "same-origin")
|
||||
.Add(Header_Twitter_Token, String.Empty)
|
||||
.Add("x-twitter-active-user", "yes")
|
||||
.Add("x-twitter-auth-type", "OAuth2Session")
|
||||
.Add(Header_Twitter_Authorization, String.Empty)
|
||||
End With
|
||||
End With
|
||||
End With
|
||||
ElseIf Site = Sites.Reddit Then
|
||||
Responser.CookiesDomain = "reddit.com"
|
||||
Responser.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
End If
|
||||
Case Sites.Reddit
|
||||
Responser.CookiesDomain = "reddit.com"
|
||||
Responser.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
Case Sites.Instagram : Responser.CookiesDomain = "instagram.com"
|
||||
End Select
|
||||
Responser.SaveSettings()
|
||||
End If
|
||||
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"),
|
||||
_XML, {SettingsCLS.Name_Node_Sites, Site.ToString}, XMLValue(Of SFile).ToFilePath)
|
||||
|
||||
Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
|
||||
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, n, XMLValue(Of SFile).ToFilePath)
|
||||
_Path.ReplaceByValue("Path", {Site.ToString})
|
||||
_XML.Remove(Site.ToString)
|
||||
|
||||
Temporary = New XMLValue(Of Boolean)
|
||||
Temporary.SetExtended("Temporary", False, _XML, n)
|
||||
Temporary.SetDefault(_Temp)
|
||||
|
||||
DownloadImages = New XMLValue(Of Boolean)
|
||||
DownloadImages.SetExtended("DownloadImages", True, _XML, n)
|
||||
DownloadImages.SetDefault(_Imgs)
|
||||
|
||||
DownloadVideos = New XMLValue(Of Boolean)
|
||||
DownloadVideos.SetExtended("DownloadVideos", True, _XML, n)
|
||||
DownloadVideos.SetDefault(_Vids)
|
||||
|
||||
If Site = Sites.Twitter Then
|
||||
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
|
||||
GetUserMediaOnly.ReplaceByValue("TwitterDefaultGetUserMedia", n)
|
||||
Else
|
||||
GetUserMediaOnly = New XMLValue(Of Boolean)
|
||||
End If
|
||||
|
||||
If Site = Sites.Instagram Then
|
||||
InstaHash = New XMLValue(Of String)("InstaHash", String.Empty, _XML, n)
|
||||
InstaHashUpdateRequired = New XMLValue(Of Boolean)("InstaHashUpdateRequired", True, _XML, n)
|
||||
If (InstaHash.IsEmptyString Or InstaHashUpdateRequired) And Responser.Cookies.ListExists Then GatherInstaHash()
|
||||
InstagramDownloadingErrorDate = New XMLValue(Of Date) With {.ToStringFunction = Function(ss, vv) AConvert(Of String)(vv, Nothing)}
|
||||
InstagramDownloadingErrorDate.SetExtended("InstagramDownloadingErrorDate", Now.AddYears(-10), _XML, n)
|
||||
Else
|
||||
InstaHash = New XMLValue(Of String)
|
||||
InstaHashUpdateRequired = New XMLValue(Of Boolean)
|
||||
End If
|
||||
If Site = Sites.Reddit Then
|
||||
SavedPostsUserName = New XMLValue(Of String)("SavedPostsUserName", String.Empty, _XML, n)
|
||||
Else
|
||||
SavedPostsUserName = New XMLValue(Of String)
|
||||
End If
|
||||
End Sub
|
||||
Friend Sub Update()
|
||||
Responser.SaveSettings()
|
||||
End Sub
|
||||
Friend Function GatherInstaHash() As Boolean
|
||||
Try
|
||||
Dim rs As New RegexStructure("=" & Chr(34) & "([^" & Chr(34) & "]+?ConsumerLibCommons[^" & Chr(34) & "]+?.js)" & Chr(34), 1) With {
|
||||
.UseTimeOut = True,
|
||||
.MatchTimeOutSeconds = 10
|
||||
}
|
||||
Dim r$ = Responser.GetResponse("https://instagram.com",, EDP.ThrowException)
|
||||
If Not r.IsEmptyString Then
|
||||
Dim hStr$ = RegexReplace(r, rs)
|
||||
If Not hStr.IsEmptyString Then
|
||||
Do While Left(hStr, 1) = "/" : hStr = Right(hStr, hStr.Length - 1) : Loop
|
||||
hStr = $"https://instagram.com/{hStr}"
|
||||
r = Responser.GetResponse(hStr,, EDP.ThrowException)
|
||||
If Not r.IsEmptyString Then
|
||||
rs = New RegexStructure("generatePaginationActionCreators.+?.profilePosts.byUserId.get.+?queryId:.([\d\w\S]+?)" & Chr(34), 1) With {
|
||||
.UseTimeOut = True,
|
||||
.MatchTimeOutSeconds = 10
|
||||
}
|
||||
Dim h$ = RegexReplace(r, rs)
|
||||
If Not h.IsEmptyString Then
|
||||
InstaHash.Value = h
|
||||
InstaHashUpdateRequired.Value = False
|
||||
Return True
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Return False
|
||||
Catch ex As Exception
|
||||
InstaHashUpdateRequired.Value = True
|
||||
InstaHash.Value = String.Empty
|
||||
Return ErrorsDescriber.Execute(EDP.SendInLog + EDP.ReturnValue, ex, "[SiteSettings.GaterInstaHash]", False)
|
||||
End Try
|
||||
End Function
|
||||
#Region "IDisposable Support"
|
||||
Private disposedValue As Boolean = False
|
||||
Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)
|
||||
|
||||
@@ -78,6 +78,7 @@ Namespace API.Base
|
||||
#Region "Declarations"
|
||||
Friend MustOverride Property Site As Sites Implements IContentProvider.Site
|
||||
Friend User As UserInfo
|
||||
Friend Property IsSavedPosts As Boolean
|
||||
Protected Const NonExistendUserHelp As String = "404"
|
||||
Protected Const SuspendedUserHelp As String = "403"
|
||||
Friend Overridable Property UserExists As Boolean = True Implements IUserData.Exists
|
||||
@@ -329,17 +330,16 @@ BlockNullPicture:
|
||||
Dim luv$ = String.Empty
|
||||
If LastUpdated.HasValue Then luv = $"{LastUpdated.Value.ToStringDate(ADateTime.Formats.BaseDateTime)}: "
|
||||
Return $"{luv}{Name} [{Site}]{IIf(HasError, " (with errors)", String.Empty)}: P - {_DownloadedPicturesTotal}; V - {_DownloadedVideosTotal}" &
|
||||
$" (P - {_CountPictures}; V - {_CountVideo})"
|
||||
$" (P - {_CountPictures}; V - {_CountVideo})"
|
||||
End Get
|
||||
End Property
|
||||
#End Region
|
||||
Protected ReadOnly LNC As New ListAddParams(LAP.NotContainsOnly)
|
||||
#End Region
|
||||
#Region "LVI"
|
||||
Friend ReadOnly Property LVIKey As String Implements IUserData.LVIKey
|
||||
Get
|
||||
If Not _IsCollection Then
|
||||
Return $"{IIf(Site = Sites.Reddit, "R", "T")}_{Name}"
|
||||
Return $"{Interaction.Switch(Site = Sites.Reddit, "R", Site = Sites.Twitter, "T", Site = Sites.Instagram, "I")}_{Name}"
|
||||
Else
|
||||
Return $"CCCC_{CollectionName}"
|
||||
End If
|
||||
@@ -361,14 +361,18 @@ BlockNullPicture:
|
||||
End Function
|
||||
Friend Overridable ReadOnly Property FitToAddParams As Boolean Implements IUserData.FitToAddParams
|
||||
Get
|
||||
Select Case Settings.ShowingMode.Value
|
||||
Case ShowingModes.Regular : Return Not Temporary And Not Favorite
|
||||
Case ShowingModes.Temporary : Return Temporary
|
||||
Case ShowingModes.Favorite : Return Favorite
|
||||
Case ShowingModes.Labels : Return Settings.Labels.CurrentSelection.ListContains(Labels)
|
||||
Case ShowingModes.NoLabels : Return Labels.Count = 0
|
||||
Case Else : Return True
|
||||
End Select
|
||||
If Settings.SelectedSites.Count = 0 OrElse Settings.SelectedSites.Contains(Site) Then
|
||||
Select Case Settings.ShowingMode.Value
|
||||
Case ShowingModes.Regular : Return Not Temporary And Not Favorite
|
||||
Case ShowingModes.Temporary : Return Temporary
|
||||
Case ShowingModes.Favorite : Return Favorite
|
||||
Case ShowingModes.Labels : Return Settings.Labels.CurrentSelection.ListContains(Labels)
|
||||
Case ShowingModes.NoLabels : Return Labels.Count = 0
|
||||
Case Else : Return True
|
||||
End Select
|
||||
Else
|
||||
Return False
|
||||
End If
|
||||
End Get
|
||||
End Property
|
||||
Friend Function GetLVIGroup(ByVal Destination As ListView) As ListViewGroup Implements IUserData.GetLVIGroup
|
||||
@@ -383,7 +387,7 @@ BlockNullPicture:
|
||||
Return Destination.Groups.Item(LabelsKeeper.NoLabeledName)
|
||||
End If
|
||||
Else
|
||||
Return Destination.Groups.Item(GetLviGroupName(Site, Temporary, Favorite, IsCollection))
|
||||
Return Destination.Groups.Item(GetLviGroupName(Site, Temporary, Favorite, IsCollection, IsChannel))
|
||||
End If
|
||||
Catch ex As Exception
|
||||
Return Destination.Groups.Item(LabelsKeeper.NoLabeledName)
|
||||
@@ -431,6 +435,7 @@ BlockNullPicture:
|
||||
Return New Reddit.UserData(u, _LoadUserInformation)
|
||||
End If
|
||||
Case Sites.Twitter : Return New Twitter.UserData(u, _LoadUserInformation)
|
||||
Case Sites.Instagram : Return New Instagram.UserData(u, _LoadUserInformation)
|
||||
Case Else : Throw New ArgumentOutOfRangeException("Site", $"Site [{u.Site}] information does not recognized by loader")
|
||||
End Select
|
||||
End Function
|
||||
@@ -503,7 +508,7 @@ BlockNullPicture:
|
||||
|
||||
x.Save(MyFile)
|
||||
End Using
|
||||
Settings.UpdateUsersList(User)
|
||||
If Not IsSavedPosts Then Settings.UpdateUsersList(User)
|
||||
Catch ex As Exception
|
||||
LogError(ex, "user information saving error")
|
||||
End Try
|
||||
@@ -584,7 +589,8 @@ BlockNullPicture:
|
||||
Select Case Site
|
||||
Case Sites.Reddit : URL = $"https://www.reddit.com/{IIf(IsChannel, "r", "user")}/{Name}/"
|
||||
Case Sites.Twitter : URL = $"https://twitter.com/{Name}"
|
||||
Case Else : MsgBoxE($"Site [{Site}] opening does not implemented", MsgBoxStyle.Exclamation)
|
||||
Case Sites.Instagram : URL = $"https://www.instagram.com/{Name}/"
|
||||
Case Else : MsgBoxE($"Site [{Site}] opening not implemented", MsgBoxStyle.Exclamation)
|
||||
End Select
|
||||
If Not URL.IsEmptyString Then Process.Start(URL)
|
||||
Catch ex As Exception
|
||||
@@ -604,10 +610,12 @@ BlockNullPicture:
|
||||
UpdateDataFiles()
|
||||
If Not Responser Is Nothing Then Responser.Dispose()
|
||||
Responser = New PersonalUtilities.Tools.WEB.Response
|
||||
Responser.Copy(Settings.Site(Site).Responser)
|
||||
Responser.Copy(Settings(Site).Responser)
|
||||
Dim UpPic As Boolean = Settings.ViewModeIsPicture AndAlso GetPicture(False) Is Nothing
|
||||
Dim sEnvir() As Boolean = {UserExists, UserSuspended}
|
||||
Dim EnvirChanged As Func(Of Boolean) = Function() Not sEnvir(0) = UserExists Or Not sEnvir(1) = UserSuspended
|
||||
UserExists = True
|
||||
UserSuspended = False
|
||||
_DownloadedPicturesSession = 0
|
||||
_DownloadedVideosSession = 0
|
||||
_TempMediaList.Clear()
|
||||
@@ -714,7 +722,7 @@ BlockNullPicture:
|
||||
End Function
|
||||
Friend Function DeleteF(ByVal Instance As IUserData) As Integer
|
||||
Dim f As SFile = SFile.GetPath(MyFile.CutPath.Path)
|
||||
If f.Exists(SFO.Path, False) AndAlso f.Delete(SFO.Path, False, False) Then
|
||||
If f.Exists(SFO.Path, False) AndAlso (User.Merged OrElse f.Delete(SFO.Path, False, False)) Then
|
||||
ImageHandler(Me, False)
|
||||
Settings.UsersList.Remove(User)
|
||||
Settings.UpdateUsersList()
|
||||
@@ -919,7 +927,7 @@ BlockNullPicture:
|
||||
#End Region
|
||||
#Region "IEquatable Support"
|
||||
Friend Overridable Overloads Function Equals(ByVal Other As UserDataBase) As Boolean Implements IEquatable(Of UserDataBase).Equals
|
||||
Return Site = Other.Site And Name = Other.Name
|
||||
Return Site = Other.Site And Name = Other.Name And IsSavedPosts = Other.IsSavedPosts
|
||||
End Function
|
||||
Public Overrides Function Equals(ByVal Obj As Object) As Boolean
|
||||
If TypeOf Obj Is Reddit.Channel Then
|
||||
|
||||
69
SCrawler/API/Imgur/Envir.vb
Normal file
69
SCrawler/API/Imgur/Envir.vb
Normal file
@@ -0,0 +1,69 @@
|
||||
' Copyright (C) 2022 Andy
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Tools.WebDocuments.JSON
|
||||
Imports System.Net
|
||||
Imports SCrawler.API.Imgur.Declarations
|
||||
Namespace API.Imgur.Declarations
|
||||
Friend Module Imgur_Declarations
|
||||
Friend ReadOnly PostRegex As New RegexStructure("/([\w\d]+?)(|\.[\w]{0,4})\Z", 1)
|
||||
End Module
|
||||
End Namespace
|
||||
Namespace API.Imgur
|
||||
Friend NotInheritable Class Envir
|
||||
Private Sub New()
|
||||
End Sub
|
||||
Friend Shared Function GetGallery(ByVal URL As String) As List(Of String)
|
||||
Try
|
||||
If Not Settings.ImgurClientID.IsEmptyString And Not URL.IsEmptyString Then
|
||||
Dim __url$ = RegexReplace(URL, PostRegex)
|
||||
If Not __url.IsEmptyString Then
|
||||
__url = $"https://api.imgur.com/post/v1/albums/{__url}?client_id={Settings.ImgurClientID.Value}&include=media"
|
||||
Using w As New WebClient
|
||||
Dim r$ = w.DownloadString(__url)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing
|
||||
If j.Contains("media") Then
|
||||
Dim UrlsList As New List(Of String)
|
||||
Dim tmpUrl$
|
||||
For Each m As EContainer In j("media")
|
||||
tmpUrl = m.Value("url")
|
||||
If Not tmpUrl.IsEmptyString Then UrlsList.ListAddValue(tmpUrl, Base.LNC)
|
||||
Next
|
||||
Return UrlsList
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
Return Nothing
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.ReturnValue + EDP.SendInLog, ex, $"[API.Imgur.Envir.GetGallery({URL})]", Nothing)
|
||||
End Try
|
||||
End Function
|
||||
Friend Shared Function GetImage(ByVal URL As String) As String
|
||||
Try
|
||||
If Not Settings.ImgurClientID.IsEmptyString And Not URL.IsEmptyString Then
|
||||
Dim __url$ = RegexReplace(URL, PostRegex)
|
||||
If Not __url.IsEmptyString Then
|
||||
__url = $"https://api.imgur.com/3/image/{__url}?client_id={Settings.ImgurClientID.Value}&include=media"
|
||||
Using w As New WebClient
|
||||
Dim r$ = w.DownloadString(__url)
|
||||
If Not r.IsEmptyString Then Return JsonDocument.Parse(r).XmlIfNothing.Value({"data"}, "link")
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
Return String.Empty
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.ReturnValue + EDP.SendInLog, ex, $"[API.Imgur.Envir.GetImage({URL})]", String.Empty)
|
||||
End Try
|
||||
End Function
|
||||
End Class
|
||||
End Namespace
|
||||
23
SCrawler/API/Instagram/Declarations.vb
Normal file
23
SCrawler/API/Instagram/Declarations.vb
Normal file
@@ -0,0 +1,23 @@
|
||||
' Copyright (C) 2022 Andy
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Namespace API.Instagram
|
||||
Friend Module Declarations
|
||||
Friend ReadOnly FilesPattern As New RegexStructure("[^\./]+?\.\w+", True, False, 2,,,, String.Empty, EDP.ReturnValue)
|
||||
Friend ReadOnly Property DateProvider As New JsonDate
|
||||
Friend Class JsonDate : Implements ICustomProvider
|
||||
Friend Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider,
|
||||
Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object Implements ICustomProvider.Convert
|
||||
Return ADateTime.ParseUnicode(Value)
|
||||
End Function
|
||||
Private Function GetFormat(ByVal FormatType As Type) As Object Implements IFormatProvider.GetFormat
|
||||
Throw New NotImplementedException("GetFormat is not available in this context")
|
||||
End Function
|
||||
End Class
|
||||
End Module
|
||||
End Namespace
|
||||
241
SCrawler/API/Instagram/UserData.vb
Normal file
241
SCrawler/API/Instagram/UserData.vb
Normal file
@@ -0,0 +1,241 @@
|
||||
' Copyright (C) 2022 Andy
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Tools.WebDocuments.JSON
|
||||
Imports SCrawler.API.Base
|
||||
Imports System.Threading
|
||||
Imports System.Net
|
||||
Imports UStates = SCrawler.API.Base.UserMedia.States
|
||||
Imports UTypes = SCrawler.API.Base.UserMedia.Types
|
||||
Namespace API.Instagram
|
||||
Friend Class UserData : Inherits UserDataBase
|
||||
Friend Overrides Property Site As Sites = Sites.Instagram
|
||||
''' <summary>Video downloader initializer</summary>
|
||||
Private Sub New()
|
||||
End Sub
|
||||
''' <summary>Default initializer</summary>
|
||||
Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True)
|
||||
User = u
|
||||
If _LoadUserInformation Then LoadUserInformation()
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
DownloadData(String.Empty, Token)
|
||||
End Sub
|
||||
Private _InstaHash As String = String.Empty
|
||||
Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
Dim n As EContainer, nn As EContainer, node As EContainer
|
||||
Dim HasNextPage As Boolean = False
|
||||
Dim EndCursor$ = String.Empty
|
||||
Dim PostID$ = String.Empty, PostDate$ = String.Empty
|
||||
|
||||
'Check environment
|
||||
If Cursor.IsEmptyString And _InstaHash.IsEmptyString Then _InstaHash = Settings(Sites.Instagram).InstaHash
|
||||
If _InstaHash.IsEmptyString Then Throw New ArgumentNullException("InstHash", "Query hash is null")
|
||||
If ID.IsEmptyString Then GetUserId()
|
||||
If ID.IsEmptyString Then Throw New ArgumentException("User ID is not detected", "ID")
|
||||
|
||||
'Create query
|
||||
Dim vars$ = "{""id"":" & ID & ",""first"":12,""after"":""" & Cursor & """}"
|
||||
vars = SymbolsConverter.ASCII.EncodeSymbolsOnly(vars)
|
||||
URL = $"https://www.instagram.com/graphql/query/?query_hash={_InstaHash}&variables={vars}"
|
||||
|
||||
'Get response
|
||||
Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
|
||||
Settings(Sites.Instagram).InstagramTooManyRequests(False)
|
||||
ThrowAny(Token)
|
||||
|
||||
'Data
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing
|
||||
n = j.ItemF({"data", "user", 0}).XmlIfNothing
|
||||
If n.Count > 0 Then
|
||||
If n.Contains("page_info") Then
|
||||
With n("page_info")
|
||||
HasNextPage = .Value("has_next_page").FromXML(Of Boolean)(False)
|
||||
EndCursor = .Value("end_cursor")
|
||||
End With
|
||||
End If
|
||||
n = n("edges").XmlIfNothing
|
||||
If n.Count > 0 Then
|
||||
For Each nn In n
|
||||
ThrowAny(Token)
|
||||
node = nn(0).XmlIfNothing
|
||||
|
||||
PostID = node.Value("id")
|
||||
If Not PostID.IsEmptyString And _TempPostsList.Contains(PostID) Then Exit Sub
|
||||
_TempPostsList.Add(PostID)
|
||||
PostDate = node.Value("taken_at_timestamp")
|
||||
|
||||
ObtainMedia(node, PostID, PostDate)
|
||||
Next
|
||||
End If
|
||||
Else
|
||||
If j.Value("status") = "ok" AndAlso j({"data", "user"}).XmlIfNothing.Count = 0 AndAlso _TempMediaList.Count = 0 Then
|
||||
Settings(Sites.Instagram).InstaHashUpdateRequired.Value = True
|
||||
UserExists = False
|
||||
Exit Sub
|
||||
End If
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
If HasNextPage And Not EndCursor.IsEmptyString Then DownloadData(EndCursor, Token)
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
If Responser.StatusCode = HttpStatusCode.NotFound Then
|
||||
UserExists = False
|
||||
ElseIf Responser.StatusCode = HttpStatusCode.BadRequest Then
|
||||
MyMainLOG = "Instagram credentials have expired"
|
||||
Settings(Sites.Instagram).InstaHashUpdateRequired.Value = True
|
||||
ElseIf Responser.StatusCode = 429 Then
|
||||
Settings(Sites.Instagram).InstagramTooManyRequests(True)
|
||||
Else
|
||||
Settings(Sites.Instagram).InstaHashUpdateRequired.Value = True
|
||||
LogError(ex, $"data downloading error [{URL}]")
|
||||
End If
|
||||
HasError = True
|
||||
Finally
|
||||
_InstaHash = String.Empty
|
||||
End Try
|
||||
End Sub
|
||||
Private Sub ObtainMedia(ByVal node As EContainer, ByVal PostID As String, ByVal PostDate As String)
|
||||
Dim CreateMedia As Action(Of EContainer) =
|
||||
Sub(ByVal e As EContainer)
|
||||
Dim t As UTypes = If(e.Value("is_video").FromXML(Of Boolean)(False), UTypes.Video, UTypes.Picture)
|
||||
Dim tmpValue$
|
||||
If t = UTypes.Picture Then
|
||||
tmpValue = e.Value("display_url")
|
||||
Else
|
||||
tmpValue = e.Value("video_url")
|
||||
End If
|
||||
If Not tmpValue.IsEmptyString Then _TempMediaList.ListAddValue(MediaFromData(t, tmpValue, PostID, PostDate), LNC)
|
||||
End Sub
|
||||
If node.Contains({"edge_sidecar_to_children", "edges"}) Then
|
||||
For Each edge As EContainer In node({"edge_sidecar_to_children", "edges"}) : CreateMedia(edge("node").XmlIfNothing) : Next
|
||||
Else
|
||||
CreateMedia(node)
|
||||
End If
|
||||
End Sub
|
||||
Private Sub GetUserId()
|
||||
Try
|
||||
Dim r$ = Responser.GetResponse($"https://www.instagram.com/{Name}/?__a=1",, EDP.ThrowException)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing
|
||||
ID = j({"graphql", "user"}, "id").XmlIfNothingValue
|
||||
End Using
|
||||
End If
|
||||
Catch ex As Exception
|
||||
If Responser.StatusCode = HttpStatusCode.NotFound Or Responser.StatusCode = HttpStatusCode.BadRequest Then
|
||||
Throw ex
|
||||
Else
|
||||
LogError(ex, "get instagram user id")
|
||||
End If
|
||||
End Try
|
||||
End Sub
|
||||
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Dim i%
|
||||
Dim dCount% = 0, dTotal% = 0
|
||||
ThrowAny(Token)
|
||||
If _ContentNew.Count > 0 Then
|
||||
_ContentNew.RemoveAll(Function(c) c.URL.IsEmptyString)
|
||||
If _ContentNew.Count > 0 Then
|
||||
MyFile.Exists(SFO.Path)
|
||||
Dim MyDir$ = MyFile.CutPath.PathNoSeparator
|
||||
Dim vsf As Boolean = SeparateVideoFolderF
|
||||
Dim f As SFile
|
||||
Dim v As UserMedia
|
||||
Using w As New WebClient
|
||||
If vsf Then SFileShares.SFileExists($"{MyDir}\Video\", SFO.Path)
|
||||
MainProgress.TotalCount += _ContentNew.Count
|
||||
For i = 0 To _ContentNew.Count - 1
|
||||
ThrowAny(Token)
|
||||
v = _ContentNew(i)
|
||||
v.State = UStates.Tried
|
||||
If v.File.IsEmptyString Then
|
||||
f = v.URL
|
||||
Else
|
||||
f = v.File
|
||||
End If
|
||||
f.Separator = "\"
|
||||
f.Path = MyDir
|
||||
|
||||
If v.URL_BASE.IsEmptyString Then v.URL_BASE = v.URL
|
||||
|
||||
If Not v.File.IsEmptyString AndAlso Not v.URL_BASE.IsEmptyString Then
|
||||
Try
|
||||
If v.Type = UTypes.Video And vsf Then f.Path = $"{f.PathWithSeparator}Video"
|
||||
w.DownloadFile(v.URL_BASE, f.ToString)
|
||||
Select Case v.Type
|
||||
Case UTypes.Video : DownloadedVideos += 1 : _CountVideo += 1
|
||||
Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
|
||||
End Select
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
v.State = UStates.Downloaded
|
||||
Catch wex As Exception
|
||||
ErrorDownloading(f, v.URL_BASE)
|
||||
End Try
|
||||
Else
|
||||
v.State = UStates.Skipped
|
||||
End If
|
||||
_ContentNew(i) = v
|
||||
If DownloadTopCount.HasValue AndAlso dCount >= DownloadTopCount.Value Then
|
||||
MainProgress.Perform(_ContentNew.Count - dTotal)
|
||||
Exit Sub
|
||||
Else
|
||||
dTotal += 1
|
||||
MainProgress.Perform()
|
||||
End If
|
||||
Next
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
LogError(ex, "content downloading error")
|
||||
HasError = True
|
||||
End Try
|
||||
End Sub
|
||||
Private Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String) As UserMedia
|
||||
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
|
||||
Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID}}
|
||||
If Not m.URL.IsEmptyString Then m.File = CStr(RegexReplace(m.URL, FilesPattern))
|
||||
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, Declarations.DateProvider, Nothing) Else m.Post.Date = Nothing
|
||||
Return m
|
||||
End Function
|
||||
Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
|
||||
Try
|
||||
If Not URL.IsEmptyString AndAlso URL.Contains("instagram.com") Then
|
||||
Do While Right(URL, 1) = "/" : URL = Left(URL, URL.Length - 1) : Loop
|
||||
URL = $"{URL}/?__a=1"
|
||||
Using t As New UserData
|
||||
t.Responser = New PersonalUtilities.Tools.WEB.Response
|
||||
t.Responser.Copy(Settings(Sites.Instagram).Responser)
|
||||
Dim r$ = t.Responser.GetResponse(URL,, EDP.ThrowException)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing
|
||||
Dim node As EContainer = j({"graphql", "shortcode_media"}).XmlIfNothing
|
||||
If node.Count > 0 Then t.ObtainMedia(node, String.Empty, String.Empty)
|
||||
End Using
|
||||
End If
|
||||
If t._TempMediaList.Count > 0 Then Return ListAddList(Nothing, t._TempMediaList)
|
||||
End Using
|
||||
End If
|
||||
Return Nothing
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.ShowMainMsg + EDP.SendInLog, ex, "Instagram standalone downloader: fetch media error")
|
||||
End Try
|
||||
End Function
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -24,7 +24,7 @@ Namespace API.Reddit
|
||||
Return ADateTime.ParseUnicodeJS(Value, NothingArg, e)
|
||||
End Function
|
||||
Private Function GetFormat(ByVal FormatType As Type) As Object Implements IFormatProvider.GetFormat
|
||||
Throw New NotImplementedException("GetFormat does not available in this context")
|
||||
Throw New NotImplementedException("GetFormat is not available in this context")
|
||||
End Function
|
||||
End Class
|
||||
Friend Class JsonDateChannel : Implements ICustomProvider
|
||||
@@ -34,7 +34,7 @@ Namespace API.Reddit
|
||||
Return ADateTime.ParseUnicode(AConvert(Of Integer)(Value, EUR_PROVIDER, Value), NothingArg, e)
|
||||
End Function
|
||||
Private Function GetFormat(ByVal FormatType As Type) As Object Implements IFormatProvider.GetFormat
|
||||
Throw New NotImplementedException("GetFormat does not available in this context")
|
||||
Throw New NotImplementedException("GetFormat is not available in this context")
|
||||
End Function
|
||||
End Class
|
||||
End Module
|
||||
|
||||
39
SCrawler/API/Reddit/ProfileSaved.vb
Normal file
39
SCrawler/API/Reddit/ProfileSaved.vb
Normal file
@@ -0,0 +1,39 @@
|
||||
' Copyright (C) 2022 Andy
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports SCrawler.API.Base
|
||||
Namespace API.Reddit
|
||||
Friend NotInheritable Class ProfileSaved
|
||||
Private Sub New()
|
||||
End Sub
|
||||
Friend Shared Sub Download(ByRef Toolbar As StatusStrip, ByRef PR As ToolStripProgressBar)
|
||||
Try
|
||||
Dim Bar = New PersonalUtilities.Forms.Toolbars.MyProgress(Toolbar, PR, Nothing)
|
||||
Dim u As New UserInfo(Settings(Sites.Reddit).SavedPostsUserName.Value, Sites.Reddit) With {
|
||||
.IsChannel = True,
|
||||
.SpecialPath = $"{Settings(Sites.Reddit).Path.PathWithSeparator}\!Saved\"
|
||||
}
|
||||
u.UpdateUserFile()
|
||||
Using user As IUserData = UserDataBase.GetInstance(u)
|
||||
DirectCast(user.Self, UserDataBase).IsSavedPosts = True
|
||||
Bar.Enabled = True
|
||||
DirectCast(user.Self, UserData).Progress = Bar
|
||||
If Not user.FileExists Then user.UpdateUserInformation()
|
||||
user.DownloadData(Nothing)
|
||||
Dim m As New MMessage("Reddit saved posts download complete", "Saved posts downloading", {"OK", "Open folder"})
|
||||
m.Text.StringAppendLine($"Downloaded images: {user.DownloadedPictures}")
|
||||
m.Text.StringAppendLine($"Downloaded videos: {user.DownloadedVideos}")
|
||||
If MsgBoxE(m) = 1 Then u.File.CutPath.Open(SFO.Path)
|
||||
Bar.Enabled = False
|
||||
End Using
|
||||
Catch ex As Exception
|
||||
ErrorsDescriber.Execute(EDP.SendInLog, ex, "[API.Reddit.ProfileSaved.Download]")
|
||||
End Try
|
||||
End Sub
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -79,7 +79,7 @@ Namespace API.Reddit
|
||||
If IsChannel AndAlso Not ChannelInfo.IsRegularChannel Then
|
||||
If Not Responser Is Nothing Then Responser.Dispose()
|
||||
Responser = New PersonalUtilities.Tools.WEB.Response
|
||||
Responser.Copy(Settings.Site(Sites.Reddit).Responser)
|
||||
Responser.Copy(Settings(Sites.Reddit).Responser)
|
||||
ChannelPostsNames.ListAddList(ChannelInfo.PostsAll.Select(Function(p) p.ID), LNC)
|
||||
If SkipExistsUsers Then _ExistsUsersNames.ListAddList(Settings.UsersList.Select(Function(p) p.Name), LNC)
|
||||
DownloadDataF(Token)
|
||||
@@ -227,7 +227,12 @@ Namespace API.Reddit
|
||||
Dim eCount As Predicate(Of EContainer) = Function(e) e.Count > 0
|
||||
Dim lDate As Date?
|
||||
|
||||
URL = $"https://reddit.com/r/{Name}/new.json?allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
|
||||
If IsSavedPosts Then
|
||||
URL = $"https://www.reddit.com/user/{Name}/saved.json?after={POST}"
|
||||
Else
|
||||
URL = $"https://reddit.com/r/{Name}/new.json?allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
|
||||
End If
|
||||
|
||||
ThrowAny(Token)
|
||||
Dim r$ = GetSiteResponse(URL)
|
||||
If Not r.IsEmptyString Then
|
||||
@@ -396,15 +401,15 @@ Namespace API.Reddit
|
||||
LogError(ex, "video reparsing error")
|
||||
End Try
|
||||
End Sub
|
||||
Friend Shared Function GetVideoInfo(ByVal URL As String) As UserMedia
|
||||
Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
|
||||
Try
|
||||
If Not URL.IsEmptyString AndAlso URL.Contains("redgifs") Then
|
||||
Using r As New UserData
|
||||
r._TempMediaList.Add(MediaFromData(UTypes.VideoPre, URL, String.Empty, String.Empty,, False))
|
||||
r.Responser = New PersonalUtilities.Tools.WEB.Response
|
||||
r.Responser.Copy(Settings.Site(Sites.Reddit).Responser)
|
||||
r.Responser.Copy(Settings(Sites.Reddit).Responser)
|
||||
r.ReparseVideo(Nothing)
|
||||
If r._TempMediaList.ListExists Then Return r._TempMediaList(0)
|
||||
If r._TempMediaList.ListExists Then Return {r._TempMediaList(0)}
|
||||
End Using
|
||||
End If
|
||||
Return Nothing
|
||||
@@ -460,13 +465,23 @@ Namespace API.Reddit
|
||||
Dim v As UserMedia
|
||||
Dim cached As Boolean = IsChannel And SaveToCache
|
||||
Dim vsf As Boolean = SeparateVideoFolderF
|
||||
Dim ImgFormat As Imaging.ImageFormat
|
||||
Dim UseMD5 As Boolean = Not IsChannel Or (Not cached And Settings.ChannelsRegularCheckMD5)
|
||||
Dim bDP As New ErrorsDescriber(EDP.None)
|
||||
Dim ImgurUrls As New List(Of String)
|
||||
Dim TryBytes As Func(Of String, Imaging.ImageFormat, String) =
|
||||
Function(ByVal __URL As String, ByVal ImgFormat As Imaging.ImageFormat) As String
|
||||
Try
|
||||
Return ByteArrayToString(GetMD5(SFile.GetBytesFromNet(__URL, bDP), ImgFormat))
|
||||
Catch hash_ex As Exception
|
||||
Return String.Empty
|
||||
End Try
|
||||
End Function
|
||||
Dim MD5BS As Func(Of String, UTypes,
|
||||
SFile, Boolean, String) = Function(ByVal __URL As String, ByVal __MT As UTypes,
|
||||
ByVal __File As SFile, ByVal __IsBase As Boolean) As String
|
||||
Try
|
||||
ImgurUrls.Clear()
|
||||
Dim ImgFormat As Imaging.ImageFormat
|
||||
If __MT = UTypes.GIF Then
|
||||
ImgFormat = Imaging.ImageFormat.Gif
|
||||
ElseIf __IsBase Then
|
||||
@@ -474,7 +489,27 @@ Namespace API.Reddit
|
||||
Else
|
||||
ImgFormat = GetImageFormat(__File)
|
||||
End If
|
||||
Return ByteArrayToString(GetMD5(SFile.GetBytesFromNet(__URL, bDP), ImgFormat))
|
||||
|
||||
Dim tmpBytes$ = TryBytes(__URL, ImgFormat)
|
||||
If tmpBytes.IsEmptyString And Not __MT = UTypes.GIF Then
|
||||
ImgFormat = Imaging.ImageFormat.Png
|
||||
tmpBytes = TryBytes(__URL, ImgFormat)
|
||||
If Not tmpBytes.IsEmptyString Then Return tmpBytes
|
||||
Else
|
||||
Return tmpBytes
|
||||
End If
|
||||
|
||||
If tmpBytes.IsEmptyString And Not __MT = UTypes.GIF And __URL.Contains("imgur.com") Then
|
||||
For c% = 0 To 1
|
||||
If c = 0 Then
|
||||
ImgurUrls.ListAddList(Imgur.Envir.GetGallery(__URL))
|
||||
Else
|
||||
ImgurUrls.ListAddValue(Imgur.Envir.GetImage(__URL))
|
||||
End If
|
||||
If ImgurUrls.Count > 0 Then Exit For
|
||||
Next
|
||||
End If
|
||||
Return tmpBytes
|
||||
Catch hash_ex As Exception
|
||||
Return String.Empty
|
||||
End Try
|
||||
@@ -496,41 +531,57 @@ Namespace API.Reddit
|
||||
m = String.Empty
|
||||
If (v.Type = UTypes.Picture Or v.Type = UTypes.GIF) And UseMD5 Then
|
||||
m = MD5BS(v.URL, v.Type, f, False)
|
||||
If m.IsEmptyString AndAlso Not v.URL_BASE.IsEmptyString AndAlso Not v.URL_BASE = v.URL Then
|
||||
If ImgurUrls.Count = 0 AndAlso m.IsEmptyString AndAlso Not v.URL_BASE.IsEmptyString AndAlso Not v.URL_BASE = v.URL Then
|
||||
m = MD5BS(v.URL_BASE, v.Type, f, True)
|
||||
If Not m.IsEmptyString Then v.URL = v.URL_BASE
|
||||
End If
|
||||
End If
|
||||
|
||||
If (Not m.IsEmptyString AndAlso Not HashList.Contains(m)) Or Not (v.Type = UTypes.Picture Or
|
||||
v.Type = UTypes.GIF) Or Not UseMD5 Then
|
||||
If Not cached Then HashList.Add(m)
|
||||
v.MD5 = m
|
||||
f.Path = MyDir
|
||||
Try
|
||||
If (v.Type = UTypes.Video Or v.Type = UTypes.m3u8) And vsf Then f.Path = $"{f.PathWithSeparator}Video"
|
||||
If v.Type = UTypes.m3u8 Then
|
||||
f = M3U8.Download(v.URL, f)
|
||||
Else
|
||||
w.DownloadFile(v.URL, f.ToString)
|
||||
End If
|
||||
If Not v.Type = UTypes.m3u8 Or Not f.IsEmptyString Then
|
||||
Select Case v.Type
|
||||
Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
|
||||
Case UTypes.Video, UTypes.m3u8 : DownloadedVideos += 1 : _CountVideo += 1
|
||||
End Select
|
||||
If Not IsChannel Or Not SaveToCache Then
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
Else
|
||||
v.File = f
|
||||
v.Type = UTypes.GIF) Or Not UseMD5 Or ImgurUrls.Count > 0 Then
|
||||
Do
|
||||
If Not cached And Not m.IsEmptyString Then HashList.Add(m)
|
||||
v.MD5 = m
|
||||
If ImgurUrls.Count > 0 Then
|
||||
If ImgurUrls(0).IsEmptyString Then ImgurUrls.RemoveAt(0) : Continue Do
|
||||
f = UrlToFile(ImgurUrls(0))
|
||||
If f.Extension.IsEmptyString Then f.Extension = "gif"
|
||||
If f.Name.IsEmptyString Then
|
||||
f.Path = MyDir
|
||||
f.Name = $"ImgurImg_{v.File.Name}"
|
||||
f = SFile.Indexed_IndexFile(f,,, EDP.ReturnValue)
|
||||
End If
|
||||
v.Post.CachedFile = f
|
||||
v.State = UStates.Downloaded
|
||||
dCount += 1
|
||||
End If
|
||||
Catch wex As Exception
|
||||
If Not IsChannel Then ErrorDownloading(f, v.URL)
|
||||
End Try
|
||||
f.Path = MyDir
|
||||
Try
|
||||
If (v.Type = UTypes.Video Or v.Type = UTypes.m3u8 Or (ImgurUrls.Count > 0 AndAlso f.Extension = "mp4")) And
|
||||
vsf Then f.Path = $"{f.PathWithSeparator}Video"
|
||||
If v.Type = UTypes.m3u8 Then
|
||||
f = M3U8.Download(v.URL, f)
|
||||
ElseIf ImgurUrls.Count > 0 Then
|
||||
w.DownloadFile(ImgurUrls(0), f.ToString)
|
||||
Else
|
||||
w.DownloadFile(v.URL, f.ToString)
|
||||
End If
|
||||
If Not v.Type = UTypes.m3u8 Or Not f.IsEmptyString Then
|
||||
Select Case v.Type
|
||||
Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
|
||||
Case UTypes.Video, UTypes.m3u8 : DownloadedVideos += 1 : _CountVideo += 1
|
||||
End Select
|
||||
If Not IsChannel Or Not SaveToCache Then
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
Else
|
||||
v.File = f
|
||||
End If
|
||||
v.Post.CachedFile = f
|
||||
v.State = UStates.Downloaded
|
||||
dCount += 1
|
||||
End If
|
||||
Catch wex As Exception
|
||||
If Not IsChannel Then ErrorDownloading(f, v.URL)
|
||||
End Try
|
||||
If ImgurUrls.Count > 0 Then ImgurUrls.RemoveAt(0)
|
||||
Loop While ImgurUrls.Count > 0
|
||||
Else
|
||||
v.State = UStates.Skipped
|
||||
End If
|
||||
|
||||
@@ -130,18 +130,18 @@ Namespace API.Twitter
|
||||
End If
|
||||
End Try
|
||||
End Sub
|
||||
Friend Shared Function GetVideoInfo(ByVal URL As String) As UserMedia
|
||||
Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
|
||||
Try
|
||||
If URL.Contains("twitter") Then
|
||||
Dim PostID$ = RegexReplace(URL, New RegexStructure("(?<=/)\d+", True, False,,,,, String.Empty))
|
||||
If Not PostID.IsEmptyString Then
|
||||
Dim r$ = DirectCast(Settings.Site(Sites.Twitter).Responser.Copy(), Response).
|
||||
Dim r$ = DirectCast(Settings(Sites.Twitter).Responser.Copy(), Response).
|
||||
GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r)
|
||||
If j.ListExists Then
|
||||
Dim u$ = GetVideoNodeURL(j)
|
||||
If Not u.IsEmptyString Then Return MediaFromData(u, PostID, String.Empty)
|
||||
If Not u.IsEmptyString Then Return {MediaFromData(u, PostID, String.Empty)}
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
@@ -224,12 +224,13 @@ Namespace API.Twitter
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Dim i%
|
||||
Dim dCount% = 0, dTotal% = 0
|
||||
ThrowAny(Token)
|
||||
If _ContentNew.Count > 0 Then
|
||||
_ContentNew.RemoveAll(Function(c) c.URL.IsEmptyString)
|
||||
If _ContentNew.Count > 0 Then
|
||||
MyFile.Exists(SFO.Path)
|
||||
Dim MyDir$ = MyFile.CutPath.Path
|
||||
Dim MyDir$ = MyFile.CutPath.PathNoSeparator
|
||||
Dim vsf As Boolean = SeparateVideoFolderF
|
||||
Dim f As SFile
|
||||
Dim v As UserMedia
|
||||
@@ -260,6 +261,7 @@ Namespace API.Twitter
|
||||
End Select
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
v.State = UStates.Downloaded
|
||||
dCount += 1
|
||||
Catch wex As Exception
|
||||
ErrorDownloading(f, v.URL_BASE)
|
||||
End Try
|
||||
@@ -267,7 +269,13 @@ Namespace API.Twitter
|
||||
v.State = UStates.Skipped
|
||||
End If
|
||||
_ContentNew(i) = v
|
||||
MainProgress.Perform()
|
||||
If DownloadTopCount.HasValue AndAlso dCount >= DownloadTopCount.Value Then
|
||||
MainProgress.Perform(_ContentNew.Count - dTotal)
|
||||
Exit Sub
|
||||
Else
|
||||
dTotal += 1
|
||||
MainProgress.Perform()
|
||||
End If
|
||||
Next
|
||||
End Using
|
||||
End If
|
||||
|
||||
@@ -407,7 +407,7 @@ Namespace API
|
||||
Dim f As SFile
|
||||
If MsgBoxE({$"Collection may contain data{vbCr}Do you really want to delete collection and all of it files?", "Collection deleting"},
|
||||
MsgBoxStyle.Exclamation + MsgBoxStyle.YesNo) = MsgBoxResult.Yes Then
|
||||
f = Collections(0).File.CutPath(2).PathWithSeparator
|
||||
f = Collections(0).File.CutPath(IIf(DataMerging, 1, 2)).PathWithSeparator
|
||||
Settings.Users.Remove(Me)
|
||||
Collections.ForEach(Sub(c) c.Delete())
|
||||
Downloader.UserRemove(Me)
|
||||
|
||||
Reference in New Issue
Block a user