mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-16 00:32:18 +00:00
2.0.0.4
Fixed network paths bug Updated classes for new library structures and functions Fixed some typos Fixed minor bugs SitePaths
This commit is contained in:
Andy
@@ -7,13 +7,14 @@
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports PersonalUtilities.Functions.XML.Base
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Namespace API.Reddit
|
||||
Friend Module Declarations
|
||||
Friend ReadOnly JsonNodesJson() As NodeParams = {New NodeParams("posts", True, True, True, True, 3)}
|
||||
Friend ReadOnly ChannelJsonNodes() As NodeParams = {New NodeParams("data", True, True, True, True, 1),
|
||||
New NodeParams("children", True, True, True)}
|
||||
Friend ReadOnly UrlBasePattern As New RegexStructure("(?<=/)([^/]+?\.[\w]{3,4})(?=(\?|\Z))", True, False)
|
||||
Friend ReadOnly VideoRegEx As New RegexStructure("http.{0,1}://[^" & Chr(34) & "]+?mp4", True, False)
|
||||
Friend ReadOnly UrlBasePattern As RParams = RParams.DM("(?<=/)([^/]+?\.[\w]{3,4})(?=(\?|\Z))", 0)
|
||||
Friend ReadOnly VideoRegEx As RParams = RParams.DM("http.{0,1}://[^" & Chr(34) & "]+?mp4", 0)
|
||||
Friend ReadOnly DateProvider As New JsonDate
|
||||
Friend ReadOnly DateProviderChannel As New JsonDateChannel
|
||||
Private ReadOnly EUR_PROVIDER As New ANumbers(ANumbers.Cultures.EUR)
|
||||
|
||||
@@ -9,26 +9,25 @@
|
||||
Imports System.Net
|
||||
Imports SCrawler.API.Reddit.M3U8_Declarations
|
||||
Imports PersonalUtilities.Tools.WEB
|
||||
Namespace API.Reddit.M3U8_Declarations
|
||||
Friend Module M3U8_Declarations
|
||||
Friend ReadOnly BaseUrlPattern As New RegexStructure("([htps:/]{7,8}.+?/.+?)(?=/)", True, False,,,,,, EDP.ReturnValue)
|
||||
Friend ReadOnly PlayListRegEx_1 As New RegexStructure("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", True, False,,,
|
||||
RegexReturn.List,, New List(Of String),
|
||||
New ErrorsDescriber(False, False, True, New List(Of String)))
|
||||
Friend ReadOnly PlayListRegEx_2 As New RegexStructure("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", True, False,,, RegexReturn.List,,
|
||||
New List(Of String),
|
||||
New ErrorsDescriber(False, False, True, New List(Of String)))
|
||||
Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
|
||||
End Module
|
||||
End Namespace
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Namespace API.Reddit
|
||||
Namespace M3U8_Declarations
|
||||
Friend Module M3U8_Declarations
|
||||
Friend ReadOnly BaseUrlPattern As RParams = RParams.DM("([htps:/]{7,8}.+?/.+?)(?=/)", 0, EDP.ReturnValue)
|
||||
Friend ReadOnly PlayListRegEx_1 As RParams = RParams.DM("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", 0,
|
||||
RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
|
||||
Friend ReadOnly PlayListRegEx_2 As RParams = RParams.DM("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", 0,
|
||||
RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
|
||||
Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
|
||||
End Module
|
||||
End Namespace
|
||||
Friend NotInheritable Class M3U8
|
||||
Private Sub New()
|
||||
End Sub
|
||||
Private Structure Resolution : Implements IRegExCreator, IComparable(Of Resolution)
|
||||
Friend File As String
|
||||
Friend Resolution As Integer
|
||||
Friend Function CreateFromArray(ByVal ParamsArray() As String) As IRegExCreator Implements IRegExCreator.CreateFromArray
|
||||
Friend Function CreateFromArray(ByVal ParamsArray() As String) As Object Implements IRegExCreator.CreateFromArray
|
||||
If ParamsArray.ArrayExists Then
|
||||
File = ParamsArray(0)
|
||||
If ParamsArray.Length > 1 Then Resolution = AConvert(Of Integer)(ParamsArray(1), 0)
|
||||
@@ -45,7 +44,7 @@ Namespace API.Reddit
|
||||
Using w As New WebClient
|
||||
Dim r$ = w.DownloadString(PlayListURL)
|
||||
If Not r.IsEmptyString Then
|
||||
Dim l As List(Of Resolution) = RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
|
||||
Dim l As List(Of Resolution) = FNF.RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
|
||||
If l.ListExists Then
|
||||
l.Sort()
|
||||
Dim pls$ = $"{BaseUrl}/{l.First.File}"
|
||||
|
||||
@@ -11,7 +11,7 @@ Imports System.Threading
|
||||
Imports PersonalUtilities.Forms.Toolbars
|
||||
Namespace API.Reddit
|
||||
Friend NotInheritable Class ProfileSaved
|
||||
Friend Shared ReadOnly Property DataPath As SFile = $"{Settings(Sites.Reddit).Path.PathNoSeparator}\!Saved\"
|
||||
Friend Shared ReadOnly Property DataPath As SFile = Settings(Sites.Reddit).SavedPostsPath
|
||||
Private Sub New()
|
||||
End Sub
|
||||
Friend Shared Sub Download(ByRef Bar As MyProgress, ByVal Token As CancellationToken)
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports PersonalUtilities.Tools.ImageRenderer
|
||||
Imports PersonalUtilities.Tools.WebDocuments.JSON
|
||||
Imports System.Net
|
||||
@@ -72,6 +73,7 @@ Namespace API.Reddit
|
||||
#End Region
|
||||
#Region "Download Overrides"
|
||||
Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
|
||||
_CrossPosts.Clear()
|
||||
If Not IsSavedPosts AndAlso (IsChannel AndAlso Not ChannelInfo.IsRegularChannel) Then
|
||||
If Not Responser Is Nothing Then Responser.Dispose()
|
||||
Responser = New PersonalUtilities.Tools.WEB.Response
|
||||
@@ -282,6 +284,18 @@ Namespace API.Reddit
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
ElseIf Not s.Value({"media", "reddit_video"}, "fallback_url").IsEmptyString Then
|
||||
tmpUrl = s.Value({"media", "reddit_video"}, "fallback_url")
|
||||
If SaveToCache Then
|
||||
tmpUrl = s.Value("thumbnail")
|
||||
If Not tmpUrl.IsEmptyString Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
Else
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre + UTypes.m3u8, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
ElseIf CreateImgurMedia(tmpUrl, PostID, PostDate, _UserID, IsChannel) Then
|
||||
_TotalPostsDownloaded += 1
|
||||
ElseIf s.Item("media_metadata").XmlIfNothing.Count > 0 Then
|
||||
@@ -370,15 +384,16 @@ Namespace API.Reddit
|
||||
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
|
||||
Try
|
||||
ThrowAny(Token)
|
||||
If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(p) p.Type = UTypes.VideoPre) Then
|
||||
Const v2 As UTypes = UTypes.VideoPre + UTypes.m3u8
|
||||
If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(p) p.Type = UTypes.VideoPre Or p.Type = v2) Then
|
||||
Dim r$, v$
|
||||
Dim e As New ErrorsDescriber(EDP.ReturnValue)
|
||||
Dim m As UserMedia
|
||||
For i% = _TempMediaList.Count - 1 To 0 Step -1
|
||||
ThrowAny(Token)
|
||||
If _TempMediaList(i).Type = UTypes.VideoPre Then
|
||||
If _TempMediaList(i).Type = UTypes.VideoPre Or _TempMediaList(i).Type = v2 Then
|
||||
m = _TempMediaList(i)
|
||||
r = Responser.GetResponse(m.URL,, e)
|
||||
If _TempMediaList(i).Type = UTypes.VideoPre Then r = Responser.GetResponse(m.URL,, e) Else r = m.URL
|
||||
_TempMediaList(i) = New UserMedia
|
||||
If Not r.IsEmptyString Then
|
||||
v = RegexReplace(r, VideoRegEx)
|
||||
@@ -440,6 +455,8 @@ Namespace API.Reddit
|
||||
#End Region
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Const _RFN$ = "RedditVideo"
|
||||
Const RFN$ = _RFN & "{0}"
|
||||
Dim i%
|
||||
Dim dCount% = 0, dTotal% = 0
|
||||
ThrowAny(Token)
|
||||
@@ -453,6 +470,10 @@ Namespace API.Reddit
|
||||
Else
|
||||
MyDir = MyFile.CutPath.PathNoSeparator
|
||||
End If
|
||||
Dim StartRFN% = 0
|
||||
If _ContentNew.Exists(Function(c) c.Type = UTypes.Video And c.URL.Contains("redd.it")) Then
|
||||
StartRFN = SFile.Indexed_GetMaxIndex($"{MyDir}\{IIf(SeparateVideoFolderF, "Video\", String.Empty)}{_RFN}.mp4",, New SFileNumbers(_RFN, String.Empty), EDP.ReturnValue)
|
||||
End If
|
||||
Dim HashList As New List(Of String)
|
||||
If _ContentList.Count > 0 Then HashList.ListAddList((From h In _ContentList Where Not h.MD5.IsEmptyString Select h.MD5), LNC)
|
||||
Dim f As SFile
|
||||
@@ -550,6 +571,10 @@ Namespace API.Reddit
|
||||
Try
|
||||
If (v.Type = UTypes.Video Or v.Type = UTypes.m3u8 Or (ImgurUrls.Count > 0 AndAlso f.Extension = "mp4")) And
|
||||
vsf Then f.Path = $"{f.PathWithSeparator}Video"
|
||||
If v.Type = UTypes.Video AndAlso v.URL.Contains("redd.it") Then
|
||||
StartRFN += 1
|
||||
f.Name = String.Format(RFN, StartRFN)
|
||||
End If
|
||||
If v.Type = UTypes.m3u8 Then
|
||||
f = M3U8.Download(v.URL, f)
|
||||
ElseIf ImgurUrls.Count > 0 Then
|
||||
|
||||
Reference in New Issue
Block a user