Fixed network paths bug
Updated classes for new library structures and functions
Fixed some typos
Fixed minor bugs
SitePaths
This commit is contained in:
Andy
2022-02-07 09:43:35 +03:00
parent 1bc048578d
commit 5b64e5563b
24 changed files with 256 additions and 81 deletions

View File

@@ -9,6 +9,7 @@
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.XML.Base
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Base
Friend Class SiteSettings : Implements IDisposable
Friend Const Header_Twitter_Authorization As String = "authorization"
@@ -16,13 +17,33 @@ Namespace API.Base
Friend ReadOnly Site As Sites
Friend ReadOnly Responser As WEB.Response
Private ReadOnly _Path As XMLValue(Of SFile)
Friend Property Path As SFile
Friend Property Path(Optional ByVal SetProp As Boolean = True) As SFile
Get
If _Path.IsEmptyString Then _Path.Value = SFile.GetPath($"{Settings.GlobalPath.Value.PathWithSeparator}{Site}")
If _Path.IsEmptyString Then
Dim tmpPath As SFile = SFile.GetPath($"{Settings.GlobalPath.Value.PathWithSeparator}{Site}")
If SetProp Then _Path.Value = tmpPath Else Return tmpPath
End If
Return _Path.Value
End Get
Set(ByVal NewFile As SFile)
_Path.Value = NewFile
Set(ByVal NewPath As SFile)
_Path.Value = NewPath
End Set
End Property
Private ReadOnly _SavedPostsPath As XMLValue(Of SFile)
Friend Property SavedPostsPath(Optional ByVal GetAny As Boolean = True) As SFile
Get
If Not _SavedPostsPath.Value.IsEmptyString Then
Return _SavedPostsPath.Value
Else
If GetAny Then
Return $"{Path.PathNoSeparator}\!Saved\"
Else
Return Nothing
End If
End If
End Get
Set(ByVal NewPath As SFile)
_SavedPostsPath.Value = NewPath
End Set
End Property
#Region "Instagram"
@@ -122,7 +143,7 @@ Namespace API.Base
End If
Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, n, XMLValue(Of SFile).ToFilePath)
_Path = New XMLValue(Of SFile)("Path",, _XML, n, XMLValue(Of SFile).ToFilePath)
_XML.Remove(Site.ToString)
Temporary = New XMLValue(Of Boolean)
@@ -138,6 +159,7 @@ Namespace API.Base
DownloadVideos.SetDefault(_Vids)
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
_SavedPostsPath = New XMLValue(Of SFile)("SavedPostsPath",, _XML, n, XMLValue(Of SFile).ToFilePath)
CreateProp(InstaHashUpdateRequired, Sites.Instagram, "InstaHashUpdateRequired", True, _XML, n)
CreateProp(InstaHash, Sites.Instagram, "InstaHash", String.Empty, _XML, n)
@@ -149,7 +171,7 @@ Namespace API.Base
CreateProp(RequestsWaitTimerTaskCount, Sites.Instagram, "RequestsWaitTimerTaskCount", 1, _XML, n)
CreateProp(SleepTimerOnPostsLimit, Sites.Instagram, "SleepTimerOnPostsLimit", 60000, _XML, n)
If Site = Sites.Instagram Then
InstagramDownloadingErrorDate = New XMLValue(Of Date) With {.ToStringFunction = Function(ss, vv) AConvert(Of String)(vv, Nothing)}
InstagramDownloadingErrorDate = New XMLValue(Of Date) With {.ToStringFunction = Function(ss, vv) AConvert(Of String)(vv, AModes.Var, Nothing)}
InstagramDownloadingErrorDate.SetExtended("InstagramDownloadingErrorDate", Now.AddYears(-10), _XML, n)
Else
InstagramDownloadingErrorDate = New XMLValue(Of Date)
@@ -170,10 +192,7 @@ Namespace API.Base
End Sub
Friend Function GatherInstaHash() As Boolean
Try
Dim rs As New RegexStructure("=" & Chr(34) & "([^" & Chr(34) & "]+?ConsumerLibCommons[^" & Chr(34) & "]+?.js)" & Chr(34), 1) With {
.UseTimeOut = True,
.MatchTimeOutSeconds = 10
}
Dim rs As New RParams("=""([^""]+?ConsumerLibCommons[^""]+?.js)""", Nothing, 1) With {.MatchTimeOut = 10}
Dim r$ = Responser.GetResponse("https://instagram.com",, EDP.ThrowException)
If Not r.IsEmptyString Then
Dim hStr$ = RegexReplace(r, rs)
@@ -182,10 +201,7 @@ Namespace API.Base
hStr = $"https://instagram.com/{hStr}"
r = Responser.GetResponse(hStr,, EDP.ThrowException)
If Not r.IsEmptyString Then
rs = New RegexStructure("generatePaginationActionCreators.+?.profilePosts.byUserId.get.+?queryId:.([\d\w\S]+?)" & Chr(34), 1) With {
.UseTimeOut = True,
.MatchTimeOutSeconds = 10
}
rs = New RParams("generatePaginationActionCreators.+?.profilePosts.byUserId.get.+?queryId:.([\d\w\S]+?)""", Nothing, 1) With {.MatchTimeOut = 10}
Dim h$ = RegexReplace(r, rs)
If Not h.IsEmptyString Then
InstaHash.Value = h

View File

@@ -7,6 +7,7 @@
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Forms.Toolbars
Imports System.IO
Imports System.Net
@@ -470,7 +471,7 @@ BlockNullPicture:
Temporary = x.Value(Name_Temporary).FromXML(Of Boolean)(False)
Favorite = x.Value(Name_Favorite).FromXML(Of Boolean)(False)
CreatedByChannel = x.Value(Name_CreatedByChannel).FromXML(Of Boolean)(False)
SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), AModes.Var, Nothing)
ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
DownloadVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
@@ -944,7 +945,7 @@ BlockNullPicture:
End Sub
Private Function CheckFile(ByVal f As SFile, ByRef List As IEnumerable(Of SFile)) As SFile
If List.ListExists Then
Dim p As New RegexStructure(".+?\s{0,1}\((\d+)\)|.+",,,,,,, String.Empty, EDP.ReturnValue)
Dim p As RParams = RParams.DMS(".+?\s{0,1}\((\d+)\)|.+", 0, EDP.ReturnValue)
Dim i% = List.Where(Function(ff) CStr(RegexReplace(ff.Name, p)).Trim.ToLower = f.Name.Trim.ToLower).Count
If i > 0 Then f.Name &= $" ({i + 1})"
End If

View File

@@ -7,16 +7,17 @@
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools.WebDocuments.JSON
Imports System.Net
Imports SCrawler.API.Imgur.Declarations
Imports SCrawler.API.Base
Namespace API.Imgur.Declarations
Friend Module Imgur_Declarations
Friend ReadOnly PostRegex As New RegexStructure("/([\w\d]+?)(|\.[\w]{0,4})\Z", 1)
End Module
End Namespace
Namespace API.Imgur
Namespace Declarations
Friend Module Imgur_Declarations
Friend ReadOnly PostRegex As RParams = RParams.DMS("/([\w\d]+?)(|\.[\w]{0,4})\Z", 1)
End Module
End Namespace
Friend NotInheritable Class Envir
Private Sub New()
End Sub

View File

@@ -6,9 +6,10 @@
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Instagram
Friend Module Declarations
Friend ReadOnly FilesPattern As New RegexStructure(".+?([^/\?]+?\.[\w\d]{3,4})(?=(\?|\Z))",,,, 1,,, String.Empty, EDP.ReturnValue)
Friend ReadOnly FilesPattern As RParams = RParams.DMS(".+?([^/\?]+?\.[\w\d]{3,4})(?=(\?|\Z))", 1, EDP.ReturnValue)
Friend ReadOnly Property DateProvider As New JsonDate
Friend Class JsonDate : Implements ICustomProvider
Friend Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider,

View File

@@ -11,7 +11,7 @@ Imports System.Threading
Imports PersonalUtilities.Forms.Toolbars
Namespace API.Instagram
Friend NotInheritable Class ProfileSaved
Friend Shared ReadOnly Property DataPath As SFile = $"{Settings(Sites.Instagram).Path.PathNoSeparator}\!Saved\"
Friend Shared ReadOnly Property DataPath As SFile = Settings(Sites.Instagram).SavedPostsPath
Private Sub New()
End Sub
Friend Shared Sub Download(ByRef Bar As MyProgress, ByVal Token As CancellationToken)

View File

@@ -8,6 +8,7 @@
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.Messaging
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools.WebDocuments.JSON
Imports PersonalUtilities.Forms.Toolbars
Imports SCrawler.API.Base
@@ -414,7 +415,7 @@ Namespace API.Instagram
Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
Try
If Not URL.IsEmptyString AndAlso URL.Contains("instagram.com") Then
Dim PID$ = RegexReplace(URL, New RegexStructure(".*?instagram.com/p/([_\w\d]+)", 1))
Dim PID$ = RegexReplace(URL, RParams.DMS(".*?instagram.com/p/([_\w\d]+)", 1))
If Not PID.IsEmptyString Then
Using t As New UserData
t.Responser = New PersonalUtilities.Tools.WEB.Response

View File

@@ -7,13 +7,14 @@
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.XML.Base
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Reddit
Friend Module Declarations
Friend ReadOnly JsonNodesJson() As NodeParams = {New NodeParams("posts", True, True, True, True, 3)}
Friend ReadOnly ChannelJsonNodes() As NodeParams = {New NodeParams("data", True, True, True, True, 1),
New NodeParams("children", True, True, True)}
Friend ReadOnly UrlBasePattern As New RegexStructure("(?<=/)([^/]+?\.[\w]{3,4})(?=(\?|\Z))", True, False)
Friend ReadOnly VideoRegEx As New RegexStructure("http.{0,1}://[^" & Chr(34) & "]+?mp4", True, False)
Friend ReadOnly UrlBasePattern As RParams = RParams.DM("(?<=/)([^/]+?\.[\w]{3,4})(?=(\?|\Z))", 0)
Friend ReadOnly VideoRegEx As RParams = RParams.DM("http.{0,1}://[^" & Chr(34) & "]+?mp4", 0)
Friend ReadOnly DateProvider As New JsonDate
Friend ReadOnly DateProviderChannel As New JsonDateChannel
Private ReadOnly EUR_PROVIDER As New ANumbers(ANumbers.Cultures.EUR)

View File

@@ -9,26 +9,25 @@
Imports System.Net
Imports SCrawler.API.Reddit.M3U8_Declarations
Imports PersonalUtilities.Tools.WEB
Namespace API.Reddit.M3U8_Declarations
Friend Module M3U8_Declarations
Friend ReadOnly BaseUrlPattern As New RegexStructure("([htps:/]{7,8}.+?/.+?)(?=/)", True, False,,,,,, EDP.ReturnValue)
Friend ReadOnly PlayListRegEx_1 As New RegexStructure("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", True, False,,,
RegexReturn.List,, New List(Of String),
New ErrorsDescriber(False, False, True, New List(Of String)))
Friend ReadOnly PlayListRegEx_2 As New RegexStructure("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", True, False,,, RegexReturn.List,,
New List(Of String),
New ErrorsDescriber(False, False, True, New List(Of String)))
Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
End Module
End Namespace
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Reddit
Namespace M3U8_Declarations
Friend Module M3U8_Declarations
Friend ReadOnly BaseUrlPattern As RParams = RParams.DM("([htps:/]{7,8}.+?/.+?)(?=/)", 0, EDP.ReturnValue)
Friend ReadOnly PlayListRegEx_1 As RParams = RParams.DM("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", 0,
RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
Friend ReadOnly PlayListRegEx_2 As RParams = RParams.DM("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", 0,
RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
End Module
End Namespace
Friend NotInheritable Class M3U8
Private Sub New()
End Sub
Private Structure Resolution : Implements IRegExCreator, IComparable(Of Resolution)
Friend File As String
Friend Resolution As Integer
Friend Function CreateFromArray(ByVal ParamsArray() As String) As IRegExCreator Implements IRegExCreator.CreateFromArray
Friend Function CreateFromArray(ByVal ParamsArray() As String) As Object Implements IRegExCreator.CreateFromArray
If ParamsArray.ArrayExists Then
File = ParamsArray(0)
If ParamsArray.Length > 1 Then Resolution = AConvert(Of Integer)(ParamsArray(1), 0)
@@ -45,7 +44,7 @@ Namespace API.Reddit
Using w As New WebClient
Dim r$ = w.DownloadString(PlayListURL)
If Not r.IsEmptyString Then
Dim l As List(Of Resolution) = RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
Dim l As List(Of Resolution) = FNF.RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
If l.ListExists Then
l.Sort()
Dim pls$ = $"{BaseUrl}/{l.First.File}"

View File

@@ -11,7 +11,7 @@ Imports System.Threading
Imports PersonalUtilities.Forms.Toolbars
Namespace API.Reddit
Friend NotInheritable Class ProfileSaved
Friend Shared ReadOnly Property DataPath As SFile = $"{Settings(Sites.Reddit).Path.PathNoSeparator}\!Saved\"
Friend Shared ReadOnly Property DataPath As SFile = Settings(Sites.Reddit).SavedPostsPath
Private Sub New()
End Sub
Friend Shared Sub Download(ByRef Bar As MyProgress, ByVal Token As CancellationToken)

View File

@@ -7,6 +7,7 @@
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools.ImageRenderer
Imports PersonalUtilities.Tools.WebDocuments.JSON
Imports System.Net
@@ -72,6 +73,7 @@ Namespace API.Reddit
#End Region
#Region "Download Overrides"
Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
_CrossPosts.Clear()
If Not IsSavedPosts AndAlso (IsChannel AndAlso Not ChannelInfo.IsRegularChannel) Then
If Not Responser Is Nothing Then Responser.Dispose()
Responser = New PersonalUtilities.Tools.WEB.Response
@@ -282,6 +284,18 @@ Namespace API.Reddit
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
_TotalPostsDownloaded += 1
End If
ElseIf Not s.Value({"media", "reddit_video"}, "fallback_url").IsEmptyString Then
tmpUrl = s.Value({"media", "reddit_video"}, "fallback_url")
If SaveToCache Then
tmpUrl = s.Value("thumbnail")
If Not tmpUrl.IsEmptyString Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
_TotalPostsDownloaded += 1
End If
Else
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre + UTypes.m3u8, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
_TotalPostsDownloaded += 1
End If
ElseIf CreateImgurMedia(tmpUrl, PostID, PostDate, _UserID, IsChannel) Then
_TotalPostsDownloaded += 1
ElseIf s.Item("media_metadata").XmlIfNothing.Count > 0 Then
@@ -370,15 +384,16 @@ Namespace API.Reddit
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
Try
ThrowAny(Token)
If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(p) p.Type = UTypes.VideoPre) Then
Const v2 As UTypes = UTypes.VideoPre + UTypes.m3u8
If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(p) p.Type = UTypes.VideoPre Or p.Type = v2) Then
Dim r$, v$
Dim e As New ErrorsDescriber(EDP.ReturnValue)
Dim m As UserMedia
For i% = _TempMediaList.Count - 1 To 0 Step -1
ThrowAny(Token)
If _TempMediaList(i).Type = UTypes.VideoPre Then
If _TempMediaList(i).Type = UTypes.VideoPre Or _TempMediaList(i).Type = v2 Then
m = _TempMediaList(i)
r = Responser.GetResponse(m.URL,, e)
If _TempMediaList(i).Type = UTypes.VideoPre Then r = Responser.GetResponse(m.URL,, e) Else r = m.URL
_TempMediaList(i) = New UserMedia
If Not r.IsEmptyString Then
v = RegexReplace(r, VideoRegEx)
@@ -440,6 +455,8 @@ Namespace API.Reddit
#End Region
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
Try
Const _RFN$ = "RedditVideo"
Const RFN$ = _RFN & "{0}"
Dim i%
Dim dCount% = 0, dTotal% = 0
ThrowAny(Token)
@@ -453,6 +470,10 @@ Namespace API.Reddit
Else
MyDir = MyFile.CutPath.PathNoSeparator
End If
Dim StartRFN% = 0
If _ContentNew.Exists(Function(c) c.Type = UTypes.Video And c.URL.Contains("redd.it")) Then
StartRFN = SFile.Indexed_GetMaxIndex($"{MyDir}\{IIf(SeparateVideoFolderF, "Video\", String.Empty)}{_RFN}.mp4",, New SFileNumbers(_RFN, String.Empty), EDP.ReturnValue)
End If
Dim HashList As New List(Of String)
If _ContentList.Count > 0 Then HashList.ListAddList((From h In _ContentList Where Not h.MD5.IsEmptyString Select h.MD5), LNC)
Dim f As SFile
@@ -550,6 +571,10 @@ Namespace API.Reddit
Try
If (v.Type = UTypes.Video Or v.Type = UTypes.m3u8 Or (ImgurUrls.Count > 0 AndAlso f.Extension = "mp4")) And
vsf Then f.Path = $"{f.PathWithSeparator}Video"
If v.Type = UTypes.Video AndAlso v.URL.Contains("redd.it") Then
StartRFN += 1
f.Name = String.Format(RFN, StartRFN)
End If
If v.Type = UTypes.m3u8 Then
f = M3U8.Download(v.URL, f)
ElseIf ImgurUrls.Count > 0 Then

View File

@@ -7,6 +7,7 @@
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools.WebDocuments.JSON
Imports System.Threading
Imports System.Net

View File

@@ -7,11 +7,12 @@
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.XML.Base
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Twitter
Friend Module Declarations
Friend DateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
Friend ReadOnly VideoNode As NodeParams() = {New NodeParams("video_info", True, True, True, True, 10)}
Friend ReadOnly VideoSizeRegEx As New RegexStructure("\d+x(\d+)",,,, 1,,, String.Empty, EDP.ReturnValue)
Friend ReadOnly UserIdRegEx As New RegexStructure("user_id.:.(\d+)",,,, 1,,, String.Empty, EDP.ReturnValue)
Friend ReadOnly VideoSizeRegEx As RParams = RParams.DMS("\d+x(\d+)", 1, EDP.ReturnValue)
Friend ReadOnly UserIdRegEx As RParams = RParams.DMS("user_id.:.(\d+)", 1, EDP.ReturnValue)
End Module
End Namespace

View File

@@ -9,6 +9,7 @@
Imports PersonalUtilities.Tools.WEB
Imports PersonalUtilities.Tools.WebDocuments.JSON
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports System.Net
Imports System.Threading
Imports SCrawler.API.Base
@@ -101,7 +102,7 @@ Namespace API.Twitter
Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
Try
If URL.Contains("twitter") Then
Dim PostID$ = RegexReplace(URL, New RegexStructure("(?<=/)\d+", True, False,,,,, String.Empty))
Dim PostID$ = RegexReplace(URL, RParams.DM("(?<=/)\d+", 0))
If Not PostID.IsEmptyString Then
Dim r$ = DirectCast(Settings(Sites.Twitter).Responser.Copy(), Response).
GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",, EDP.ReturnValue)