mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-16 00:32:18 +00:00
2022.6.3.0
Updated plugin environments and dependencies Added automation Fixed Insta hash issue Updated groups Added toast notifications Updated tagged posts notifications Updated M3U8; fixed audio issue Extended some of log exceptions Fixed minor bugs Other minor improvements
This commit is contained in:
Andy
@@ -14,23 +14,32 @@ Namespace API.Reddit
|
||||
Namespace M3U8_Declarations
|
||||
Friend Module M3U8_Declarations
|
||||
Friend ReadOnly BaseUrlPattern As RParams = RParams.DM("([htps:/]{7,8}.+?/.+?)(?=/)", 0, EDP.ReturnValue)
|
||||
''' <summary>Video</summary>
|
||||
Friend ReadOnly PlayListRegEx_1 As RParams = RParams.DM("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", 0,
|
||||
RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
|
||||
''' <summary>Audio, Video</summary>
|
||||
Friend ReadOnly PlayListRegEx_2 As RParams = RParams.DM("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", 0,
|
||||
RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
|
||||
Friend ReadOnly PlayListAudioRegEx As RParams = RParams.DM("(HLS_AUDIO_(\d+)[^""]+)", 0, RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
|
||||
Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
|
||||
End Module
|
||||
End Namespace
|
||||
Friend NotInheritable Class M3U8
|
||||
Private Sub New()
|
||||
End Sub
|
||||
Friend NotInheritable Class M3U8 : Implements IDisposable
|
||||
#Region "Declarations"
|
||||
Private Enum Types : Video : Audio : End Enum
|
||||
Private Structure Resolution : Implements IRegExCreator, IComparable(Of Resolution)
|
||||
Friend File As String
|
||||
Friend Resolution As Integer
|
||||
Friend HasError As Boolean
|
||||
Friend Function CreateFromArray(ByVal ParamsArray() As String) As Object Implements IRegExCreator.CreateFromArray
|
||||
If ParamsArray.ArrayExists Then
|
||||
File = ParamsArray(0)
|
||||
If ParamsArray.Length > 1 Then Resolution = AConvert(Of Integer)(ParamsArray(1), 0)
|
||||
Try
|
||||
If ParamsArray.Length > 1 Then Resolution = AConvert(Of Integer)(ParamsArray(1), EDP.ThrowException)
|
||||
Catch ex As Exception
|
||||
HasError = True
|
||||
Resolution = 0
|
||||
End Try
|
||||
End If
|
||||
Return Me
|
||||
End Function
|
||||
@@ -38,20 +47,59 @@ Namespace API.Reddit
|
||||
Return Resolution.CompareTo(Other.Resolution) * -1
|
||||
End Function
|
||||
End Structure
|
||||
Private Shared Function GetPlaylistUrls(ByVal PlayListURL As String, ByVal BaseUrl As String) As List(Of String)
|
||||
Private ReadOnly PlayListURL As String
|
||||
Private ReadOnly BaseURL As String
|
||||
Private ReadOnly Video As List(Of String)
|
||||
Private ReadOnly Audio As List(Of String)
|
||||
Private OutFile As SFile
|
||||
Private VideoFile As SFile
|
||||
Private AudioFile As SFile
|
||||
Private CachePath As SFile
|
||||
#End Region
|
||||
Private Sub New(ByVal URL As String, ByVal OutFile As SFile)
|
||||
PlayListURL = URL
|
||||
BaseURL = RegexReplace(URL, BaseUrlPattern)
|
||||
Video = New List(Of String)
|
||||
Audio = New List(Of String)
|
||||
Me.OutFile = OutFile
|
||||
Me.OutFile.Name = "PlayListFile"
|
||||
Me.OutFile.Extension = "mp4"
|
||||
CachePath = $"{OutFile.PathWithSeparator}_Cache\{SFile.GetDirectories($"{OutFile.PathWithSeparator}_Cache\",,, EDP.ReturnValue).ListIfNothing.Count + 1}\"
|
||||
End Sub
|
||||
#Region "Internal functions"
|
||||
#Region "GetPlaylistUrls"
|
||||
Private Overloads Sub GetPlaylistUrls()
|
||||
Video.ListAddList(GetPlaylistUrls(PlayListURL, Types.Video))
|
||||
Audio.ListAddList(GetPlaylistUrls(PlayListURL, Types.Audio))
|
||||
End Sub
|
||||
Private Overloads Function GetPlaylistUrls(ByVal PlayListURL As String, ByVal Type As Types) As List(Of String)
|
||||
Try
|
||||
If Not BaseUrl.IsEmptyString Then
|
||||
If Not BaseURL.IsEmptyString Then
|
||||
Using w As New WebClient
|
||||
Dim r$ = w.DownloadString(PlayListURL)
|
||||
If Not r.IsEmptyString Then
|
||||
Dim l As List(Of Resolution) = FNF.RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
|
||||
Dim l As New List(Of Resolution)
|
||||
If Type = Types.Video Then
|
||||
l = FNF.RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
|
||||
Else
|
||||
Try
|
||||
l = FNF.RegexFields(Of Resolution)(r, {PlayListAudioRegEx}, {1, 2})
|
||||
Catch anull As FNF.RegexFieldsTextBecameNullException
|
||||
l.Clear()
|
||||
End Try
|
||||
End If
|
||||
If l.ListExists Then
|
||||
Dim plError As Predicate(Of Resolution) = Function(lr) lr.HasError
|
||||
If l.Exists(plError) Then
|
||||
l.RemoveAll(plError)
|
||||
If l.Count = 0 Then Return New List(Of String)
|
||||
End If
|
||||
l.Sort()
|
||||
Dim pls$ = $"{BaseUrl}/{l.First.File}"
|
||||
Dim pls$ = $"{BaseURL}/{l.First.File}"
|
||||
r = w.DownloadString(pls)
|
||||
If Not r.IsEmptyString Then
|
||||
Dim lp As New ListAddParams(LAP.NotContainsOnly) With {
|
||||
.Converter = Function(input) $"{BaseUrl}/{input}",
|
||||
.Converter = Function(input) $"{BaseURL}/{input}",
|
||||
.Error = New ErrorsDescriber(False, False, True, New List(Of String))}
|
||||
Return ListAddList(Of String, List(Of String))(Nothing, DirectCast(RegexReplace(r, PlayListRegEx_2), List(Of String)), lp).ListIfNothing
|
||||
End If
|
||||
@@ -61,47 +109,94 @@ Namespace API.Reddit
|
||||
End If
|
||||
Return New List(Of String)
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(DPED, ex, "[M3U8.GetPlaylistUrls]", New List(Of String))
|
||||
Return ErrorsDescriber.Execute(DPED, ex, $"[M3U8.GetPlaylistUrls({Type}): {PlayListURL}]", New List(Of String))
|
||||
End Try
|
||||
End Function
|
||||
Private Shared Function Save(ByVal URLs As List(Of String), ByVal f As SFile) As SFile
|
||||
Dim CachePath As SFile = Nothing
|
||||
#End Region
|
||||
#Region "ConcatData"
|
||||
Private Overloads Sub ConcatData()
|
||||
ConcatData(Video, Types.Video, VideoFile)
|
||||
ConcatData(Audio, Types.Audio, AudioFile)
|
||||
MergeFiles()
|
||||
End Sub
|
||||
Private Overloads Sub ConcatData(ByVal Urls As List(Of String), ByVal Type As Types, ByRef TFile As SFile)
|
||||
Try
|
||||
If URLs.ListExists Then
|
||||
Dim ConcatFile As SFile = f
|
||||
ConcatFile.Name = "PlayListFile"
|
||||
ConcatFile.Extension = "mp4"
|
||||
CachePath = $"{f.PathWithSeparator}_Cache\{SFile.GetDirectories($"{f.PathWithSeparator}_Cache\",,, EDP.ReturnValue).ListIfNothing.Count + 1}\"
|
||||
If Urls.ListExists Then
|
||||
Dim ConcatFile As SFile = OutFile
|
||||
If Type = Types.Audio Then
|
||||
ConcatFile.Name &= "_AUDIO"
|
||||
ConcatFile.Extension = "aac"
|
||||
Else
|
||||
If Audio.Count > 0 Then ConcatFile.Name &= "_VIDEO"
|
||||
ConcatFile.Extension = "mp4"
|
||||
End If
|
||||
If CachePath.Exists(SFO.Path) Then
|
||||
Dim p As New SFileNumbers(ConcatFile.Name,,, New ANumbers With {.Format = ANumbers.Formats.General})
|
||||
ConcatFile = SFile.Indexed_IndexFile(ConcatFile,, p, EDP.ReturnValue)
|
||||
ConcatFile = SFile.Indexed_IndexFile(ConcatFile,, p, EDP.ThrowException) 'EDP.ReturnValue
|
||||
Dim i%
|
||||
Dim eFiles As New List(Of SFile)
|
||||
Dim dFile As SFile = CachePath
|
||||
dFile.Extension = New SFile(URLs(0)).Extension
|
||||
dFile.Extension = New SFile(Urls(0)).Extension
|
||||
If dFile.Extension.IsEmptyString Then dFile.Extension = "ts"
|
||||
Using w As New WebClient
|
||||
For i = 0 To URLs.Count - 1
|
||||
For i = 0 To Urls.Count - 1
|
||||
dFile.Name = $"ConPart_{i}"
|
||||
w.DownloadFile(URLs(i), dFile)
|
||||
w.DownloadFile(Urls(i), dFile)
|
||||
eFiles.Add(dFile)
|
||||
Next
|
||||
End Using
|
||||
f = FFMPEG.ConcatenateFiles(eFiles, Settings.FfmpegFile, ConcatFile, p, DPED)
|
||||
TFile = FFMPEG.ConcatenateFiles(eFiles, Settings.FfmpegFile, ConcatFile, p, DPED)
|
||||
eFiles.Clear()
|
||||
Return f
|
||||
End If
|
||||
End If
|
||||
Return Nothing
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(DPED, ex, "[M3U8.Save]", New SFile)
|
||||
Finally
|
||||
CachePath.Delete(SFO.Path, SFODelete.None, DPED)
|
||||
ErrorsDescriber.Execute(DPED, ex, $"[M3U8.Save({Type})]")
|
||||
End Try
|
||||
End Sub
|
||||
#End Region
|
||||
Private Sub MergeFiles()
|
||||
Try
|
||||
If Not VideoFile.IsEmptyString And Not AudioFile.IsEmptyString Then
|
||||
Dim p As New SFileNumbers(OutFile.Name,, RParams.DMS("PlayListFile_(\d*)", 1), New ANumbers With {.Format = ANumbers.Formats.General})
|
||||
OutFile = FFMPEG.MergeFiles({VideoFile, AudioFile}, Settings.FfmpegFile, OutFile, p, DPED)
|
||||
Else
|
||||
OutFile = VideoFile
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ErrorsDescriber.Execute(DPED, ex, $"[M3U8.MergeFiles]")
|
||||
End Try
|
||||
End Sub
|
||||
Friend Function Download() As SFile
|
||||
GetPlaylistUrls()
|
||||
ConcatData()
|
||||
Return OutFile
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Statics"
|
||||
Friend Shared Function Download(ByVal URL As String, ByVal f As SFile) As SFile
|
||||
Dim BaseUrl$ = RegexReplace(URL, BaseUrlPattern)
|
||||
Return Save(GetPlaylistUrls(URL, BaseUrl), f)
|
||||
Using m As New M3U8(URL, f) : Return m.Download() : End Using
|
||||
End Function
|
||||
#End Region
|
||||
#Region "IDisposable Support"
|
||||
Private disposedValue As Boolean = False
|
||||
Private Overloads Sub Dispose(ByVal disposing As Boolean)
|
||||
If Not disposedValue Then
|
||||
If disposing Then
|
||||
Video.Clear()
|
||||
Audio.Clear()
|
||||
CachePath.Delete(SFO.Path, SFODelete.None, DPED)
|
||||
End If
|
||||
disposedValue = True
|
||||
End If
|
||||
End Sub
|
||||
Protected Overrides Sub Finalize()
|
||||
Dispose(False)
|
||||
MyBase.Finalize()
|
||||
End Sub
|
||||
Friend Overloads Sub Dispose() Implements IDisposable.Dispose
|
||||
Dispose(True)
|
||||
GC.SuppressFinalize(Me)
|
||||
End Sub
|
||||
#End Region
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -28,6 +28,8 @@ Namespace API.Reddit
|
||||
End Property
|
||||
<PropertyOption(ControlText:="Saved posts user"), PXML("SavedPostsUserName")>
|
||||
Friend ReadOnly Property SavedPostsUserName As PropertyValue
|
||||
<PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos"), PXML>
|
||||
Friend ReadOnly Property UseM3U8 As PropertyValue
|
||||
Friend Overrides ReadOnly Property Responser As WEB.Response
|
||||
Friend Sub New()
|
||||
MyBase.New(RedditSite)
|
||||
@@ -43,6 +45,7 @@ Namespace API.Reddit
|
||||
End If
|
||||
End With
|
||||
SavedPostsUserName = New PropertyValue(String.Empty, GetType(String))
|
||||
UseM3U8 = New PropertyValue(True)
|
||||
UrlPatternUser = "https://www.reddit.com/user/{0}/"
|
||||
UrlPatternChannel = "https://www.reddit.com/r/{0}/"
|
||||
ImageVideoContains = "redgifs"
|
||||
@@ -75,17 +78,21 @@ Namespace API.Reddit
|
||||
Next
|
||||
Return Nothing
|
||||
End Function
|
||||
Friend Overrides Function Available(ByVal What As Download) As Boolean
|
||||
Friend Overrides Function Available(ByVal What As Download, ByVal Silent As Boolean) As Boolean
|
||||
Try
|
||||
Dim dl As List(Of DownDetector.Data) = DownDetector.GetData("reddit")
|
||||
If dl.ListExists Then
|
||||
dl = dl.Take(4).ToList
|
||||
Dim avg% = dl.Average(Function(d) d.Value)
|
||||
If avg > 100 Then
|
||||
Return MsgBoxE({"Over the past hour, Reddit has received an average of " &
|
||||
avg.NumToString(New ANumbers With {.FormatOptions = ANumbers.Options.GroupIntegral}) & " outage reports:" & vbCr &
|
||||
dl.ListToString(vbCr) & vbCr & vbCr &
|
||||
"Do you want to continue parsing Reddit data?", "There are outage reports on Reddit"}, vbYesNo) = vbYes
|
||||
If Silent Then
|
||||
Return False
|
||||
Else
|
||||
Return MsgBoxE({"Over the past hour, Reddit has received an average of " &
|
||||
avg.NumToString(New ANumbers With {.FormatOptions = ANumbers.Options.GroupIntegral}) & " outage reports:" & vbCr &
|
||||
dl.ListToString(vbCr) & vbCr & vbCr &
|
||||
"Do you want to continue parsing Reddit data?", "There are outage reports on Reddit"}, vbYesNo) = vbYes
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Return True
|
||||
|
||||
@@ -31,6 +31,11 @@ Namespace API.Reddit
|
||||
Return If(IsChannel, DateProviderChannel, DateProvider)
|
||||
End Get
|
||||
End Property
|
||||
Private ReadOnly Property UseM3U8 As Boolean
|
||||
Get
|
||||
Return Settings.UseM3U8 And CBool(DirectCast(HOST.Source, SiteSettings).UseM3U8.Value)
|
||||
End Get
|
||||
End Property
|
||||
#Region "Channels Support"
|
||||
#Region "IChannelLimits Support"
|
||||
Friend Property DownloadLimitCount As Integer? Implements IChannelLimits.DownloadLimitCount
|
||||
@@ -242,9 +247,12 @@ Namespace API.Reddit
|
||||
added = False
|
||||
End If
|
||||
Case "video"
|
||||
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
|
||||
If UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
|
||||
_PostID(), PostDate,, IsChannel), LNC)
|
||||
ElseIf Not UseM3U8 AndAlso s("fallback_url").XmlIfNothingValue("/").ToLower.Contains("mp4") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Video, s.Value("fallback_url"),
|
||||
_PostID(), PostDate,, IsChannel), LNC)
|
||||
Else
|
||||
added = False
|
||||
End If
|
||||
@@ -367,8 +375,12 @@ Namespace API.Reddit
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
ElseIf UseM3U8 AndAlso Not s.Value({"media", "reddit_video"}, "hls_url").IsEmptyString Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value({"media", "reddit_video"}, "hls_url"),
|
||||
PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
Else
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre + UTypes.m3u8, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
'_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre + UTypes.m3u8, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Video, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
ElseIf CreateImgurMedia(tmpUrl, PostID, PostDate, _UserID, IsChannel) Then
|
||||
@@ -718,7 +730,7 @@ Namespace API.Reddit
|
||||
ElseIf Responser.StatusCode = HttpStatusCode.BadGateway Or
|
||||
Responser.StatusCode = HttpStatusCode.ServiceUnavailable Or
|
||||
Responser.StatusCode = HttpStatusCode.GatewayTimeout Then
|
||||
MyMainLOG = "Reddit is currently unavailable"
|
||||
MyMainLOG = $"Reddit is currently unavailable ({ToString()})"
|
||||
Else
|
||||
If Not FromPE Then LogError(ex, Message) : HasError = True
|
||||
Return 0
|
||||
|
||||
Reference in New Issue
Block a user