Initial commit

SCrawler/API/Reddit/Channel.vb

@@ -0,0 +1,304 @@
+Imports PersonalUtilities.Tools
+Imports PersonalUtilities.Forms.Toolbars
+Imports PersonalUtilities.Functions.XML
+Imports SCrawler.API.Base
+Namespace API.Reddit
+    Friend Class Channel : Implements ICollection(Of UserPost), IEquatable(Of Channel), IComparable(Of Channel),
+            IRangeSwitcherContainer(Of UserPost), ILoaderSaver, IMyEnumerator(Of UserPost), IChannelLimits, IDisposable
+#Region "XML Nodes' Names"
+        Private Const Name_Name As String = "Name"
+        Private Const Name_ID As String = "ID"
+        Private Const Name_Date As String = "Date"
+        Private Const Name_PostsNode As String = "Posts"
+#End Region
+        Friend Const DefaultDownloadLimitCount As Integer = 1000
+        Friend Property Name As String = String.Empty
+        Friend Property ID As String = String.Empty
+        Friend ReadOnly Property CUser As UserInfo
+            Get
+                Return New UserInfo(Me)
+            End Get
+        End Property
+        Friend ReadOnly Property PostsLatest As List(Of UserPost)
+        Friend ReadOnly Property Posts As List(Of UserPost)
+        Friend ReadOnly Property PostsAll As List(Of UserPost)
+            Get
+                Return ListAddList(Nothing, Posts).ListAddList(PostsLatest).ListSort
+            End Get
+        End Property
+        Private ReadOnly Property Source As IEnumerable(Of UserPost) Implements IRangeSwitcherContainer(Of UserPost).Source
+            Get
+                Return Posts
+            End Get
+        End Property
+        Friend Property LatestParsedDate As Date? = Nothing
+        Private _Downloading As Boolean = False
+        Friend ReadOnly Property Downloading As Boolean
+            Get
+                Return _Downloading
+            End Get
+        End Property
+        Friend ReadOnly Property File As SFile
+            Get
+                Return $"{ChannelsCollection.ChannelsPath.PathWithSeparator}{ID}.xml"
+            End Get
+        End Property
+        Friend ReadOnly Property CachePath As SFile
+            Get
+                Return $"{ChannelsCollection.ChannelsPathCache.PathWithSeparator}{ID}\"
+            End Get
+        End Property
+        Friend ReadOnly Property Count As Integer Implements ICollection(Of UserPost).Count, IMyEnumerator(Of UserPost).MyEnumeratorCount
+            Get
+                Return Posts.Count
+            End Get
+        End Property
+        Default Friend ReadOnly Property Item(ByVal Index As Integer) As UserPost Implements IMyEnumerator(Of UserPost).MyEnumeratorObject
+            Get
+                Return Posts(Index)
+            End Get
+        End Property
+        Private ReadOnly Property Range As RangeSwitcher(Of UserPost)
+#Region "Limits Support"
+        Private _DownloadLimitCount As Integer? = Nothing
+        Friend Property DownloadLimitCount As Integer? Implements IChannelLimits.DownloadLimitCount
+            Get
+                If AutoGetLimits Then
+                    If LatestParsedDate.HasValue OrElse Not DownloadLimitPost.IsEmptyString Then
+                        Return Nothing
+                    ElseIf _DownloadLimitCount.HasValue Then
+                        Return _DownloadLimitCount
+                    Else
+                        Return DefaultDownloadLimitCount
+                    End If
+                Else
+                    Return _DownloadLimitCount
+                End If
+            End Get
+            Set(ByVal NewLimit As Integer?)
+                _DownloadLimitCount = NewLimit
+            End Set
+        End Property
+        Private _DownloadLimitPost As String = String.Empty
+        Friend Property DownloadLimitPost As String Implements IChannelLimits.DownloadLimitPost
+            Get
+                Dim PID$ = ListAddList(Nothing, Posts, LAP.NotContainsOnly).ListAddList(PostsLatest, LAP.NotContainsOnly).ListSort.FirstOrDefault.ID
+                If AutoGetLimits And Not PID.IsEmptyString Then
+                    Return PID
+                Else
+                    Return _DownloadLimitPost
+                End If
+            End Get
+            Set(ByVal NewLimit As String)
+                _DownloadLimitPost = NewLimit
+            End Set
+        End Property
+        Private _DownloadLimitDate As Date? = Nothing
+        Friend Property DownloadLimitDate As Date? Implements IChannelLimits.DownloadLimitDate
+            Get
+                If AutoGetLimits And LatestParsedDate.HasValue Then
+                    Return LatestParsedDate
+                Else
+                    Return _DownloadLimitDate
+                End If
+            End Get
+            Set(ByVal NewLimit As Date?)
+                _DownloadLimitDate = NewLimit
+            End Set
+        End Property
+        Friend Overloads Sub SetLimit(Optional ByVal MaxPost As String = "", Optional ByVal MaxCount As Integer? = Nothing,
+                                      Optional ByVal MinDate As Date? = Nothing) Implements IChannelLimits.SetLimit
+            DownloadLimitPost = MaxPost
+            DownloadLimitCount = MaxCount
+            DownloadLimitDate = MinDate
+        End Sub
+        Friend Overloads Sub SetLimit(ByVal Source As IChannelLimits) Implements IChannelLimits.SetLimit
+            With Source
+                DownloadLimitCount = .DownloadLimitCount
+                DownloadLimitPost = .DownloadLimitPost
+                DownloadLimitDate = .DownloadLimitDate
+                AutoGetLimits = .AutoGetLimits
+            End With
+        End Sub
+        Friend Property AutoGetLimits As Boolean = True Implements IChannelLimits.AutoGetLimits
+#End Region
+        Friend Sub New()
+            Posts = New List(Of UserPost)
+            PostsLatest = New List(Of UserPost)
+            Range = New RangeSwitcher(Of UserPost)(Me)
+        End Sub
+        Friend Sub New(ByVal f As SFile)
+            Me.New
+            LoadData(f, False)
+        End Sub
+        Public Shared Widening Operator CType(ByVal f As SFile) As Channel
+            Return New Channel(f)
+        End Operator
+        Public Overrides Function ToString() As String
+            If Not Name.IsEmptyString Then
+                Return Name
+            Else
+                Return ID
+            End If
+        End Function
+        Friend Sub Delete()
+            If File.Exists Then File.Delete()
+        End Sub
+        Friend Sub DownloadData(ByVal Token As Threading.CancellationToken, Optional ByVal SkipExists As Boolean = True,
+                                Optional ByVal p As MyProgress = Nothing)
+            Try
+                _Downloading = True
+                Using d As New UserData(CUser, False, False) With {
+                    .Progress = p,
+                    .SaveToCache = True,
+                    .SkipExistsUsers = SkipExists,
+                    .ChannelInfo = Me
+                }
+                    d.SetLimit(Me)
+                    d.DownloadData(Token)
+                    Posts.ListAddList(d.GetNewChannelPosts(), LAP.NotContainsOnly)
+                    Posts.Sort()
+                    LatestParsedDate = If(Posts.FirstOrDefault(Function(pp) pp.Date.HasValue).Date, LatestParsedDate)
+                    Token.ThrowIfCancellationRequested()
+                End Using
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested
+            Finally
+                _Downloading = False
+            End Try
+        End Sub
+#Region "ICollection Support"
+        Private ReadOnly Property IsReadOnly As Boolean = False Implements ICollection(Of UserPost).IsReadOnly
+        Friend Sub Add(ByVal _Item As UserPost) Implements ICollection(Of UserPost).Add
+            If Not Contains(_Item) Then Posts.Add(_Item)
+        End Sub
+        Friend Sub AddRange(ByVal _Items As IEnumerable(Of UserPost))
+            If _Items.ListExists Then
+                For Each i As UserPost In _Items : Add(i) : Next
+            End If
+        End Sub
+        Friend Sub Clear() Implements ICollection(Of UserPost).Clear
+            Posts.Clear()
+        End Sub
+        Friend Function Contains(ByVal _Item As UserPost) As Boolean Implements ICollection(Of UserPost).Contains
+            Return Count > 0 AndAlso Posts.Contains(_Item)
+        End Function
+        Private Sub CopyTo(ByVal _Array() As UserPost, ByVal ArrayIndex As Integer) Implements ICollection(Of UserPost).CopyTo
+            Throw New NotImplementedException()
+        End Sub
+        Friend Function Remove(ByVal _Item As UserPost) As Boolean Implements ICollection(Of UserPost).Remove
+            Return Posts.Remove(_Item)
+        End Function
+#End Region
+#Region "IEnumerable Support"
+        Friend Function GetEnumerator() As IEnumerator(Of UserPost) Implements IEnumerable(Of UserPost).GetEnumerator
+            Return New MyEnumerator(Of UserPost)(Me)
+        End Function
+        Friend Function IEnumerable_GetEnumerator() As IEnumerator Implements IEnumerable.GetEnumerator
+            Return GetEnumerator()
+        End Function
+#End Region
+#Region "IEquatable Support"
+        Friend Overloads Function Equals(ByVal Other As Channel) As Boolean Implements IEquatable(Of Channel).Equals
+            Return ID = Other.ID
+        End Function
+        Public Overloads Overrides Function Equals(ByVal Obj As Object) As Boolean
+            If Not Obj Is Nothing Then
+                If TypeOf Obj Is String Then
+                    Return ID = CStr(Obj)
+                Else
+                    Return Equals(DirectCast(Obj, Channel))
+                End If
+            Else
+                Return False
+            End If
+        End Function
+#End Region
+#Region "IComparable Support"
+        Friend Function CompareTo(ByVal Other As Channel) As Integer Implements IComparable(Of Channel).CompareTo
+            If Not Name.IsEmptyString And Not Other.Name.IsEmptyString Then
+                Return Name.CompareTo(Other.Name)
+            Else
+                Return ID.CompareTo(Other.ID)
+            End If
+        End Function
+#End Region
+#Region "IXMLContainer Support"
+        Friend Overloads Function LoadData(Optional ByVal f As SFile = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Implements ILoaderSaver.Load
+            Return LoadData(File, False, e)
+        End Function
+        Friend Overloads Function LoadData(ByVal f As SFile, ByVal PartialLoad As Boolean, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
+            If f.Exists Then
+                Using x As New XmlFile(f, ProtectionLevels.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
+                    x.LoadData()
+                    x.DefaultsLoading(False)
+                    If x.Count > 0 Then
+                        Dim XMLDateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
+                        Name = x.Value(Name_Name)
+                        ID = x.Value(Name_ID)
+                        LatestParsedDate = AConvert(Of Date)(x.Value(Name_Date), XMLDateProvider, Nothing)
+                        If Not PartialLoad Then
+                            With x(Name_PostsNode).XmlIfNothing
+                                If .Count > 0 Then .ForEach(Sub(ee) PostsLatest.Add(New UserPost With {
+                                                                        .ID = ee.Attribute(Name_ID),
+                                                                        .[Date] = AConvert(Of Date)(ee.Attribute(Name_Date).Value, XMLDateProvider, Nothing)}))
+                            End With
+                        End If
+                    End If
+                End Using
+            End If
+            Return True
+        End Function
+        Friend Overloads Function Save(Optional ByVal f As SFile = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Implements ILoaderSaver.Save
+            Dim XMLDateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
+            Using x As New XmlFile With {.AllowSameNames = True, .Name = "Channel"}
+                x.DefaultsLoading(False)
+                x.Add(Name_Name, Name)
+                x.Add(Name_ID, ID)
+                If Posts.Count > 0 Or PostsLatest.Count > 0 Then
+                    Dim tmpPostList As List(Of UserPost) = Nothing
+                    tmpPostList.ListAddList(Posts).ListAddList(PostsLatest)
+                    tmpPostList.Sort()
+                    LatestParsedDate = tmpPostList.FirstOrDefault(Function(pd) pd.Date.HasValue).Date
+                    x.Add(Name_Date, AConvert(Of String)(LatestParsedDate, XMLDateProvider, String.Empty))
+                    x.Add(Name_PostsNode, String.Empty)
+                    With x(Name_PostsNode)
+                        tmpPostList.Take(200).ToList.ForEach(Sub(p) .Add(New EContainer("Post",
+                                                                                  String.Empty,
+                                                                                  {
+                                                                                    New EAttribute(Name_ID, p.ID),
+                                                                                    New EAttribute(Name_Date, AConvert(Of String)(p.Date, XMLDateProvider, String.Empty))
+                                                                                  })
+                                                                  )
+                                                      )
+                    End With
+                    tmpPostList.Clear()
+                End If
+                x.Save(File)
+            End Using
+            Return True
+        End Function
+#End Region
+#Region "IDisposable Support"
+        Private disposedValue As Boolean = False
+        Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)
+            If Not disposedValue Then
+                If disposing Then
+                    Posts.Clear()
+                    PostsLatest.Clear()
+                    Range.Dispose()
+                    If CachePath.Exists(SFO.Path, False) Then CachePath.Delete(SFO.Path, False, False, EDP.SendInLog)
+                End If
+                disposedValue = True
+            End If
+        End Sub
+        Protected Overrides Sub Finalize()
+            Dispose(False)
+            MyBase.Finalize()
+        End Sub
+        Friend Overloads Sub Dispose() Implements IDisposable.Dispose
+            Dispose(True)
+            GC.SuppressFinalize(Me)
+        End Sub
+#End Region
+    End Class
+End Namespace

SCrawler/API/Reddit/ChannelsCollection.vb

@@ -0,0 +1,141 @@
+Imports PersonalUtilities.Tools
+Imports PersonalUtilities.Forms.Toolbars
+Imports SCrawler.API.Base
+Imports System.Threading
+Namespace API.Reddit
+    Friend Class ChannelsCollection : Implements ICollection(Of Channel), IMyEnumerator(Of Channel), IChannelLimits, IDisposable
+        Friend Shared ReadOnly Property ChannelsPath As SFile = $"{SettingsFolderName}\Channels\"
+        Friend Shared ReadOnly Property ChannelsPathCache As SFile = $"{Settings.GlobalPath.Value.PathWithSeparator}_CachedData\"
+        Private ReadOnly Channels As List(Of Channel)
+        Friend ReadOnly Property Downloading As Boolean
+            Get
+                If Count > 0 Then
+                    Return Channels.Exists(Function(c) c.Downloading)
+                Else
+                    Return False
+                End If
+            End Get
+        End Property
+#Region "Limits Support"
+        Friend Property DownloadLimitCount As Integer? Implements IChannelLimits.DownloadLimitCount
+        <Obsolete("This property cannot be used in collections", True)> Private Property DownloadLimitPost As String Implements IChannelLimits.DownloadLimitPost
+        Friend Property DownloadLimitDate As Date? Implements IChannelLimits.DownloadLimitDate
+        Friend Overloads Sub SetLimit(Optional ByVal MaxPost As String = "", Optional ByVal MaxCount As Integer? = Nothing,
+                                      Optional ByVal MinDate As Date? = Nothing) Implements IChannelLimits.SetLimit
+            'DownloadLimitPost = MaxPost
+            DownloadLimitCount = MaxCount
+            DownloadLimitDate = MinDate
+        End Sub
+        Friend Overloads Sub SetLimit(ByVal Source As IChannelLimits) Implements IChannelLimits.SetLimit
+            With Source
+                DownloadLimitCount = .DownloadLimitCount
+                DownloadLimitDate = .DownloadLimitDate
+                AutoGetLimits = .AutoGetLimits
+            End With
+        End Sub
+        Friend Property AutoGetLimits As Boolean = True Implements IChannelLimits.AutoGetLimits
+#End Region
+        Friend Sub New()
+            Channels = New List(Of Channel)
+        End Sub
+        Friend Sub Load()
+            If ChannelsPath.Exists(SFO.Path, False) Then
+                Dim files As List(Of SFile) = SFile.GetFiles(ChannelsPath, "*.xml")
+                If files.ListExists Then files.ForEach(Sub(f) Add(f))
+            End If
+        End Sub
+        Friend Sub Update()
+            If Count > 0 Then Channels.ForEach(Sub(c) c.Save())
+        End Sub
+        Friend ReadOnly Property Count As Integer Implements ICollection(Of Channel).Count, IMyEnumerator(Of Channel).MyEnumeratorCount
+            Get
+                Return Channels.Count
+            End Get
+        End Property
+        Default Friend ReadOnly Property Item(ByVal Index As Integer) As Channel Implements IMyEnumerator(Of Channel).MyEnumeratorObject
+            Get
+                Return Channels(Index)
+            End Get
+        End Property
+        ''' <exception cref="ArgumentException"></exception>
+        Friend ReadOnly Property Find(ByVal ChannelID As String) As Channel
+            Get
+                If Count > 0 Then
+                    For i% = 0 To Count - 1
+                        If Item(i).ID = ChannelID Then Return Item(i)
+                    Next
+                End If
+                Throw New ArgumentException($"Channel ID [{ChannelID}] does not found in channels collection", "ChannelID") With {.HelpLink = 1}
+                'Return Nothing
+            End Get
+        End Property
+        Friend Sub DownloadData(ByVal Token As CancellationToken, Optional ByVal SkipExists As Boolean = True,
+                                Optional ByVal p As MyProgress = Nothing)
+            Try
+                If Count > 0 Then
+                    Dim t As New List(Of Task)
+                    For Each c As Channel In Channels
+                        If Not c.Downloading Then t.Add(Task.Run(Sub()
+                                                                     c.SetLimit(Me)
+                                                                     c.DownloadData(Token, SkipExists, p)
+                                                                 End Sub))
+                    Next
+                    If t.Count > 0 Then Task.WaitAll(t.ToArray)
+                    Token.ThrowIfCancellationRequested()
+                End If
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested
+            End Try
+        End Sub
+#Region "ICollection Support"
+        Private ReadOnly Property IsReadOnly As Boolean = False Implements ICollection(Of Channel).IsReadOnly
+        Friend Sub Add(ByVal _Item As Channel) Implements ICollection(Of Channel).Add
+            If Not Contains(_Item) Then Channels.Add(_Item)
+        End Sub
+        Friend Sub Clear() Implements ICollection(Of Channel).Clear
+            Channels.ListClearDispose
+        End Sub
+        Private Sub CopyTo(ByVal _Array() As Channel, ByVal ArrayIndex As Integer) Implements ICollection(Of Channel).CopyTo
+            Throw New NotImplementedException()
+        End Sub
+        Friend Function Contains(ByVal _Item As Channel) As Boolean Implements ICollection(Of Channel).Contains
+            Return Count > 0 AndAlso Channels.Contains(_Item)
+        End Function
+        Friend Function Remove(ByVal _Item As Channel) As Boolean Implements ICollection(Of Channel).Remove
+            If Count > 0 Then
+                Dim i% = Channels.IndexOf(_Item)
+                If i >= 0 Then
+                    With Channels(i) : .Delete() : .Dispose() : End With
+                    Channels.RemoveAt(i)
+                    Return True
+                End If
+            End If
+            Return False
+        End Function
+#End Region
+#Region "IEnumerable Support"
+        Friend Function GetEnumerator() As IEnumerator(Of Channel) Implements IEnumerable(Of Channel).GetEnumerator
+            Return New MyEnumerator(Of Channel)(Me)
+        End Function
+        Private Function IEnumerable_GetEnumerator() As IEnumerator Implements IEnumerable.GetEnumerator
+            Return GetEnumerator()
+        End Function
+#End Region
+#Region "IDisposable Support"
+        Private disposedValue As Boolean = False
+        Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)
+            If Not disposedValue Then
+                If disposing Then Update() : Clear()
+                disposedValue = True
+            End If
+        End Sub
+        Protected Overrides Sub Finalize()
+            Dispose(False)
+            MyBase.Finalize()
+        End Sub
+        Friend Overloads Sub Dispose() Implements IDisposable.Dispose
+            Dispose(True)
+            GC.SuppressFinalize(Me)
+        End Sub
+#End Region
+    End Class
+End Namespace

SCrawler/API/Reddit/Declarations.vb

@@ -0,0 +1,33 @@
+Imports PersonalUtilities.Functions.XML.Base
+Namespace API.Reddit
+    Friend Module Declarations
+        Friend ReadOnly JsonNodesJson() As NodeParams = {New NodeParams("posts", True, True, True, True, 3)}
+        Friend ReadOnly ChannelJsonNodes() As NodeParams = {New NodeParams("data", True, True, True, True, 1),
+                                                            New NodeParams("children", True, True, True)}
+        Friend ReadOnly UrlBasePattern As New RegexStructure("(?<=/)([^/]+?\.[\w]{3,4})(?=(\?|\Z))", True, False)
+        Friend ReadOnly VideoRegEx As New RegexStructure("http.{0,1}://[^" & Chr(34) & "]+?mp4", True, False)
+        Friend ReadOnly DateProvider As New JsonDate
+        Friend ReadOnly DateProviderChannel As New JsonDateChannel
+        Private ReadOnly EUR_PROVIDER As New ANumbers(ANumbers.Modes.EUR)
+        Friend Class JsonDate : Implements ICustomProvider
+            ''' <inheritdoc cref="ADateTime.ParseUnicodeJS(Object, Object, ErrorsDescriber)"/>
+            Friend Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider,
+                                    Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object Implements ICustomProvider.Convert
+                Return ADateTime.ParseUnicodeJS(Value, NothingArg, e)
+            End Function
+            Private Function GetFormat(ByVal FormatType As Type) As Object Implements IFormatProvider.GetFormat
+                Throw New NotImplementedException("GetFormat does not available in this context")
+            End Function
+        End Class
+        Friend Class JsonDateChannel : Implements ICustomProvider
+            ''' <inheritdoc cref="ADateTime.ParseUnicodeJS(Object, Object, ErrorsDescriber)"/>
+            Friend Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider,
+                                    Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object Implements ICustomProvider.Convert
+                Return ADateTime.ParseUnicode(AConvert(Of Integer)(Value, EUR_PROVIDER, Value), NothingArg, e)
+            End Function
+            Private Function GetFormat(ByVal FormatType As Type) As Object Implements IFormatProvider.GetFormat
+                Throw New NotImplementedException("GetFormat does not available in this context")
+            End Function
+        End Class
+    End Module
+End Namespace

SCrawler/API/Reddit/M3U8.vb

@@ -0,0 +1,100 @@
+Imports System.Net
+Imports SCrawler.API.Reddit.M3U8_Declarations
+Imports PersonalUtilities.Tools.WEB
+Namespace API.Reddit.M3U8_Declarations
+    Friend Module M3U8_Declarations
+        Friend ReadOnly BaseUrlPattern As New RegexStructure("([htps:/]{7,8}.+?/.+?)(?=/)", True, False,,,,,, EDP.ReturnValue)
+        Friend ReadOnly PlayListRegEx_1 As New RegexStructure("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", True, False,,,
+                                                              RegexReturn.List,, New List(Of String),
+                                                              New ErrorsDescriber(False, False, True, New List(Of String)))
+        Friend ReadOnly PlayListRegEx_2 As New RegexStructure("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", True, False,,, RegexReturn.List,,
+                                                              New List(Of String),
+                                                              New ErrorsDescriber(False, False, True, New List(Of String)))
+        Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
+    End Module
+End Namespace
+Namespace API.Reddit
+    Friend NotInheritable Class M3U8
+        Private Sub New()
+        End Sub
+        Private Structure Resolution : Implements IRegExCreator, IComparable(Of Resolution)
+            Friend File As String
+            Friend Resolution As Integer
+            Friend Function CreateFromArray(ByVal ParamsArray() As String) As IRegExCreator Implements IRegExCreator.CreateFromArray
+                If ParamsArray.ArrayExists Then
+                    File = ParamsArray(0)
+                    If ParamsArray.Length > 1 Then Resolution = AConvert(Of Integer)(ParamsArray(1), 0)
+                End If
+                Return Me
+            End Function
+            Friend Function CompareTo(ByVal Other As Resolution) As Integer Implements IComparable(Of Resolution).CompareTo
+                Return Resolution.CompareTo(Other.Resolution) * -1
+            End Function
+        End Structure
+        Private Shared Function GetPlaylistUrls(ByVal PlayListURL As String, ByVal BaseUrl As String) As List(Of String)
+            Try
+                If Not BaseUrl.IsEmptyString Then
+                    Using w As New WebClient
+                        Dim r$ = w.DownloadString(PlayListURL)
+                        If Not r.IsEmptyString Then
+                            Dim l As List(Of Resolution) = RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
+                            If l.ListExists Then
+                                l.Sort()
+                                Dim pls$ = $"{BaseUrl}/{l.First.File}"
+                                r = w.DownloadString(pls)
+                                If Not r.IsEmptyString Then
+                                    Dim lp As New ListAddParams(LAP.NotContainsOnly) With {
+                                        .Converter = Function(input) $"{BaseUrl}/{input}",
+                                        .e = New ErrorsDescriber(False, False, True, New List(Of String))}
+                                    Return ListAddList(Of String, List(Of String))(Nothing, DirectCast(RegexReplace(r, PlayListRegEx_2), List(Of String)), lp).ListIfNothing
+                                End If
+                            End If
+                        End If
+                    End Using
+                End If
+                Return New List(Of String)
+            Catch ex As Exception
+                Return ErrorsDescriber.Execute(DPED, ex, "[M3U8.GetPlaylistUrls]", New List(Of String))
+            End Try
+        End Function
+        Private Shared Function Save(ByVal URLs As List(Of String), ByVal f As SFile) As SFile
+            Dim CachePath As SFile = Nothing
+            Try
+                If URLs.ListExists Then
+                    Dim ConcatFile As SFile = f
+                    ConcatFile.Name = "PlayListFile"
+                    ConcatFile.Extension = "mp4"
+                    CachePath = $"{f.PathWithSeparator}_Cache\{SFile.GetDirectories($"{f.PathWithSeparator}_Cache\",,, EDP.ReturnValue).ListIfNothing.Count + 1}\"
+                    If CachePath.Exists(SFO.Path) Then
+                        Dim p As New SFileNumbers(ConcatFile.Name,,, New ANumbers(ANumbers.Modes.USA) With {.GroupSeparator = String.Empty, .FormatMode = ANumbers.Formats.General})
+                        ConcatFile = SFile.Indexed_IndexFile(ConcatFile,, p, EDP.ReturnValue)
+                        Dim i%
+                        Dim eFiles As New List(Of SFile)
+                        Dim dFile As SFile = CachePath
+                        dFile.Extension = New SFile(URLs(0)).Extension
+                        If dFile.Extension.IsEmptyString Then dFile.Extension = "ts"
+                        Using w As New WebClient
+                            For i = 0 To URLs.Count - 1
+                                dFile.Name = $"ConPart_{i}"
+                                w.DownloadFile(URLs(i), dFile)
+                                eFiles.Add(dFile)
+                            Next
+                        End Using
+                        f = FFMPEG.ConcatenateFiles(eFiles, "ffmpeg.exe", ConcatFile, p, DPED)
+                        eFiles.Clear()
+                        Return f
+                    End If
+                End If
+                Return Nothing
+            Catch ex As Exception
+                Return ErrorsDescriber.Execute(DPED, ex, "[M3U8.Save]", New SFile)
+            Finally
+                If Not CachePath.IsEmptyString AndAlso CachePath.Exists(SFO.Path, False) Then CachePath.Delete(SFO.Path, False, False, DPED)
+            End Try
+        End Function
+        Friend Shared Function Download(ByVal URL As String, ByVal f As SFile) As SFile
+            Dim BaseUrl$ = RegexReplace(URL, BaseUrlPattern)
+            Return Save(GetPlaylistUrls(URL, BaseUrl), f)
+        End Function
+    End Class
+End Namespace

SCrawler/API/Reddit/UserData.vb

@@ -0,0 +1,468 @@
+Imports PersonalUtilities.Functions.XML
+Imports PersonalUtilities.Forms.Toolbars
+Imports PersonalUtilities.Tools.ImageRenderer
+Imports PersonalUtilities.Tools.WebDocuments.JSON
+Imports System.Net
+Imports System.Threading
+Imports SCrawler.API.Base
+Imports UStates = SCrawler.API.Base.UserMedia.States
+Imports UTypes = SCrawler.API.Base.UserMedia.Types
+Namespace API.Reddit
+    Friend Class UserData : Inherits UserDataBase : Implements IChannelData
+        Friend Overrides Property Site As Sites = Sites.Reddit
+#Region "Channels Support"
+#Region "IChannelLimits Support"
+        Friend Property DownloadLimitCount As Integer? Implements IChannelLimits.DownloadLimitCount
+        Friend Property DownloadLimitPost As String Implements IChannelLimits.DownloadLimitPost
+        Friend Property DownloadLimitDate As Date? Implements IChannelLimits.DownloadLimitDate
+        Friend Overloads Sub SetLimit(Optional ByVal MaxPost As String = "", Optional ByVal MaxCount As Integer? = Nothing,
+                                      Optional ByVal MinDate As Date? = Nothing) Implements IChannelLimits.SetLimit
+            DownloadLimitPost = MaxPost
+            DownloadLimitCount = MaxCount
+            DownloadLimitDate = MinDate
+        End Sub
+        Friend Overloads Sub SetLimit(ByVal Source As IChannelLimits) Implements IChannelLimits.SetLimit
+            With Source
+                DownloadLimitCount = .DownloadLimitCount
+                DownloadLimitPost = .DownloadLimitPost
+                DownloadLimitDate = .DownloadLimitDate
+                AutoGetLimits = .AutoGetLimits
+            End With
+        End Sub
+        Friend Property AutoGetLimits As Boolean = True Implements IChannelLimits.AutoGetLimits
+#End Region
+        Friend Property ChannelInfo As Channel
+        Private ReadOnly ChannelPostsNames As New List(Of String)
+        Friend Property SkipExistsUsers As Boolean = True Implements IChannelData.SkipExistsUsers
+        Private ReadOnly _ExistsUsersNames As List(Of String)
+        Friend Property SaveToCache As Boolean = False Implements IChannelData.SaveToCache
+        Friend Function GetNewChannelPosts() As IEnumerable(Of UserPost)
+            If _ContentNew.Count > 0 Then Return (From c As UserMedia In _ContentNew
+                                                  Where Not c.Post.CachedFile.IsEmptyString And c.State = UStates.Downloaded
+                                                  Select c.Post) Else Return Nothing
+        End Function
+#End Region
+        Private _Progress As MyProgress
+        Friend Property Progress As MyProgress
+            Get
+                If _Progress Is Nothing Then Return MainProgress Else Return _Progress
+            End Get
+            Set(ByVal p As MyProgress)
+                _Progress = p
+            End Set
+        End Property
+#Region "Initializers"
+        ''' <summary>Video downloader initializer</summary>
+        Private Sub New()
+        End Sub
+        ''' <summary>Default initializer</summary>
+        Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True, Optional ByVal InvokeImageHandler As Boolean = True)
+            MyBase.New(InvokeImageHandler)
+            ChannelPostsNames = New List(Of String)
+            _ExistsUsersNames = New List(Of String)
+            User = u
+            If _LoadUserInformation Then LoadUserInformation()
+        End Sub
+#End Region
+#Region "Download Overrides"
+        Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
+            If IsChannel Then
+                ChannelPostsNames.ListAddList(ChannelInfo.PostsAll.Select(Function(p) p.ID), LNC)
+                If SkipExistsUsers Then _ExistsUsersNames.ListAddList(Settings.UsersList.Select(Function(p) p.Name), LNC)
+                DownloadDataF(Token)
+                ReparseVideo(Token)
+                _ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
+                DownloadContent(Token)
+            Else
+                MyBase.DownloadData(Token)
+            End If
+        End Sub
+        Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
+            If IsChannel Then
+                _DownloadedChannelPosts = 0
+                DownloadDataChannel(String.Empty, Token)
+            Else
+                DownloadDataUser(String.Empty, Token)
+            End If
+        End Sub
+#End Region
+#Region "Download Functions (User, Channel)"
+        Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)
+            Dim URL$ = String.Empty
+            Try
+                Dim PostID$ = String.Empty
+                Dim PostDate$
+                Dim n As EContainer, nn As EContainer, s As EContainer
+                Dim NewPostDetected As Boolean = False
+                Dim ExistsDetected As Boolean = False
+                Dim _ItemsBefore%
+                Dim added As Boolean
+                Dim __ItemType$
+                Dim tmpType As UTypes
+                Dim CheckNode As Predicate(Of EContainer) = Function(e) e("author").XmlIfNothingValue("/").ToLower.Equals(Name.ToLower)
+                Dim UPicType As Func(Of String, UTypes) = Function(input) IIf(input = "image", UTypes.Picture, UTypes.GIF)
+
+                URL = $"https://gateway.reddit.com/desktopapi/v1/user/{Name}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
+                Token.ThrowIfCancellationRequested()
+                Dim r$ = GetSiteResponse(URL)
+                If Not r.IsEmptyString Then
+                    Using w As EContainer = JsonDocument.Parse(r)
+                        If w.Count > 0 Then
+                            n = w.GetNode(JsonNodesJson)
+                            If Not n Is Nothing AndAlso n.Count > 0 Then
+                                For Each nn In n
+                                    Token.ThrowIfCancellationRequested()
+                                    If nn.Count > 0 Then
+                                        PostID = nn.Name
+                                        If PostID.IsEmptyString AndAlso nn.Contains("id") Then PostID = nn("id").Value
+                                        If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
+                                        If Not _TempPostsList.Contains(PostID) Then
+                                            NewPostDetected = True
+                                            _TempPostsList.Add(PostID)
+                                        Else
+                                            ExistsDetected = True
+                                            Continue For
+                                        End If
+                                        If CheckNode(nn) Then
+                                            _ItemsBefore = _TempMediaList.Count
+                                            added = True
+                                            s = nn.ItemF({"source", "url"})
+                                            If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
+                                                _TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
+                                            Else
+                                                s = nn.ItemF({"media"}).XmlIfNothing
+                                                __ItemType = s("type").XmlIfNothingValue
+                                                Select Case __ItemType
+                                                    Case "gallery" : DownloadGallery(s, PostID, PostDate)
+                                                    Case "image", "gifvideo"
+                                                        If s.Contains("content") Then _
+                                                            _TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
+                                                                                                      PostID, PostDate,, IsChannel), LNC)
+                                                    Case "video"
+                                                        If s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then _
+                                                            _TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
+                                                                                                      PostID, PostDate,, IsChannel), LNC)
+                                                    Case Else : added = False
+                                                End Select
+                                            End If
+                                            If Not added Then
+                                                s = nn.ItemF({"source", "url"}).XmlIfNothing
+                                                If Not s.IsEmptyString AndAlso TryFile(s.Value) Then
+                                                    With s.Value.ToLower
+                                                        Select Case True
+                                                            Case .Contains("redgifs") : tmpType = UTypes.VideoPre
+                                                            Case .Contains("m3u8") : tmpType = UTypes.m3u8
+                                                            Case .Contains(".gif") And TryFile(s.Value) : tmpType = UTypes.GIF
+                                                            Case TryFile(s.Value) : tmpType = UTypes.Picture
+                                                            Case Else : tmpType = UTypes.Undefined
+                                                        End Select
+                                                    End With
+                                                    If Not tmpType = UTypes.Undefined Then _
+                                                        _TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
+                                                End If
+                                            End If
+                                        End If
+                                    End If
+                                Next
+                            End If
+                        End If
+                    End Using
+                    If POST.IsEmptyString And ExistsDetected Then Exit Sub
+                    If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataUser(PostID, Token)
+                End If
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested
+            Catch ex As Exception
+                LogError(ex, $"data downloading error [{URL}]")
+                HasError = True
+            End Try
+        End Sub
+        Private _DownloadedChannelPosts As Integer = 0
+        Private Sub DownloadDataChannel(ByVal POST As String, ByVal Token As CancellationToken)
+            Dim URL$ = String.Empty
+            Try
+                Dim PostID$ = String.Empty
+                Dim PostDate$, _UserID$, tmpUrl$
+                Dim n As EContainer, nn As EContainer, s As EContainer, ss As EContainer
+                Dim NewPostDetected As Boolean = False
+                Dim ExistsDetected As Boolean = False
+                Dim eCount As Predicate(Of EContainer) = Function(e) e.Count > 0
+                Dim lDate As Date?
+
+                URL = $"https://reddit.com/r/{Name}/new.json?allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
+                Token.ThrowIfCancellationRequested()
+                Dim r$ = GetSiteResponse(URL)
+                If Not r.IsEmptyString Then
+                    Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
+                        If w.Count > 0 Then
+                            n = w.GetNode(ChannelJsonNodes)
+                            If Not n Is Nothing AndAlso n.Count > 0 Then
+                                For Each nn In n
+                                    Token.ThrowIfCancellationRequested()
+                                    s = nn.ItemF({eCount})
+                                    If Not s Is Nothing AndAlso s.Count > 0 Then
+                                        PostID = s.Value("name")
+                                        If PostID.IsEmptyString AndAlso s.Contains("id") Then PostID = s("id").Value
+
+                                        If ChannelPostsNames.Contains(PostID) Then ExistsDetected = True : Continue For 'Exit Sub
+                                        If DownloadLimitCount.HasValue AndAlso _DownloadedChannelPosts >= DownloadLimitCount.Value Then Exit Sub
+                                        If Not DownloadLimitPost.IsEmptyString AndAlso DownloadLimitPost = PostID Then Exit Sub
+                                        If DownloadLimitDate.HasValue AndAlso _TempMediaList.Count > 0 Then
+                                            With (From __u In _TempMediaList Where __u.Post.Date.HasValue Select __u.Post.Date.Value)
+                                                If .Count > 0 Then lDate = .Min Else lDate = Nothing
+                                            End With
+                                            If lDate.HasValue AndAlso lDate.Value <= DownloadLimitDate.Value Then Exit Sub
+                                        End If
+                                        NewPostDetected = True
+
+                                        If s.Contains("created") Then PostDate = s("created").Value Else PostDate = String.Empty
+                                        _UserID = s.Value("author")
+
+                                        If SkipExistsUsers AndAlso _ExistsUsersNames.Count > 0 AndAlso
+                                           Not _UserID.IsEmptyString AndAlso _ExistsUsersNames.Contains(_UserID) Then Continue For
+
+                                        tmpUrl = s.Value("url")
+                                        If Not tmpUrl.IsEmptyString AndAlso tmpUrl.Contains("redgifs.com") Then
+                                            If SaveToCache Then
+                                                tmpUrl = s.Value({"media", "oembed"}, "thumbnail_url")
+                                                If Not tmpUrl.IsEmptyString Then
+                                                    _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
+                                                    _DownloadedChannelPosts += 1
+                                                End If
+                                            Else
+                                                _TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
+                                                _DownloadedChannelPosts += 1
+                                            End If
+                                        ElseIf s.Item("media_metadata").XmlIfNothing.Count > 0 Then
+                                            DownloadGallery(s, PostID, PostDate, _UserID, SaveToCache)
+                                            _DownloadedChannelPosts += 1
+                                        ElseIf s.Contains("preview") Then
+                                            ss = s.ItemF({"preview", "images", eCount, "source", "url"}).XmlIfNothing
+                                            If Not ss.Value.IsEmptyString Then
+                                                _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, ss.Value, PostID, PostDate, _UserID, IsChannel), LNC)
+                                                _DownloadedChannelPosts += 1
+                                            End If
+                                        End If
+                                    End If
+                                Next
+                            End If
+                        End If
+                    End Using
+                    If POST.IsEmptyString And ExistsDetected Then Exit Sub
+                    If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataChannel(PostID, Token)
+                End If
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested
+            Catch ex As Exception
+                LogError(ex, $"channel data downloading error [{URL}]")
+                HasError = True
+            End Try
+        End Sub
+#End Region
+#Region "Download Base Functions"
+        Private Sub DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
+                                    Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False)
+            Try
+                Dim cn$ = IIf(IsChannel, "media_metadata", "mediaMetadata")
+                If Not w Is Nothing AndAlso w(cn).XmlIfNothing.Count > 0 Then
+                    Dim t As EContainer
+                    For Each n As EContainer In w(cn)
+                        t = n.ItemF({"s", "u"})
+                        If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then
+                            _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel), LNC)
+                            If FirstOnly Then Exit For
+                        End If
+                    Next
+                End If
+            Catch ex As Exception
+                LogError(ex, "gallery parsing error")
+                HasError = True
+            End Try
+        End Sub
+        Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
+            Try
+                Token.ThrowIfCancellationRequested()
+                If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(p) p.Type = UTypes.VideoPre) Then
+                    Dim r$, v$
+                    Dim m As UserMedia
+                    For i% = _TempMediaList.Count - 1 To 0 Step -1
+                        Token.ThrowIfCancellationRequested()
+                        If _TempMediaList(i).Type = UTypes.VideoPre Then
+                            m = _TempMediaList(i)
+                            r = GetSiteResponse(m.URL)
+                            _TempMediaList(i) = New UserMedia
+                            If Not r.IsEmptyString Then
+                                v = RegexReplace(r, VideoRegEx)
+                                If Not v.IsEmptyString Then
+                                    _TempMediaList(i) = New UserMedia With {.Type = UTypes.Video, .URL = v, .File = v, .Post = m.Post}
+                                Else
+                                    _TempMediaList.RemoveAt(i)
+                                End If
+                            End If
+                        End If
+                    Next
+                End If
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested
+            Catch ex As Exception
+                LogError(ex, "video reparsing error")
+            End Try
+        End Sub
+        Friend Shared Function GetVideoInfo(ByVal URL As String) As UserMedia
+            Try
+                If Not URL.IsEmptyString AndAlso URL.Contains("redgifs") Then
+                    Using r As New UserData
+                        r._TempMediaList.Add(MediaFromData(UTypes.VideoPre, URL, String.Empty, String.Empty,, False))
+                        r.ReparseVideo(Nothing)
+                        If r._TempMediaList.ListExists Then Return r._TempMediaList(0)
+                    End Using
+                End If
+                Return Nothing
+            Catch ex As Exception
+                Return ErrorsDescriber.Execute(EDP.ShowMainMsg + EDP.SendInLog, ex, "Video searching error")
+            End Try
+        End Function
+#End Region
+#Region "Structure creator"
+        Protected Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
+                                                Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False) As UserMedia
+            If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing
+            _URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
+            Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
+            If t = UTypes.Picture Or t = UTypes.GIF Then m.File = CStr(RegexReplace(m.URL, FilesPattern)) Else m.File = Nothing
+            If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
+            If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
+            Return m
+        End Function
+        Private Function TryFile(ByVal URL As String) As Boolean
+            Try
+                If Not URL.IsEmptyString AndAlso URL.Contains(".jpg") Then
+                    Dim f As SFile = CStr(RegexReplace(URL, FilesPattern))
+                    Return Not f.IsEmptyString And Not f.File.IsEmptyString
+                End If
+                Return False
+            Catch ex As Exception
+                Return False
+            End Try
+        End Function
+#End Region
+        Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
+            Try
+                Dim i%
+                Token.ThrowIfCancellationRequested()
+                If _ContentNew.Count > 0 Then
+                    _ContentNew.RemoveAll(Function(c) c.URL.IsEmptyString)
+                    If _ContentNew.Count > 0 Then
+                        MyFile.Exists(SFO.Path)
+                        Dim MyDir$
+                        If IsChannel And SaveToCache Then
+                            MyDir = ChannelInfo.CachePath.PathNoSeparator
+                        Else
+                            MyDir = MyFile.CutPath.PathNoSeparator
+                        End If
+                        Dim HashList As New List(Of String)
+                        If _ContentList.Count > 0 Then HashList.ListAddList((From h In _ContentList Where Not h.MD5.IsEmptyString Select h.MD5), LNC)
+                        Dim f As SFile
+                        Dim v As UserMedia
+                        Dim cached As Boolean = IsChannel And SaveToCache
+                        Dim vsf As Boolean = SeparateVideoFolderF
+                        Dim ImgFormat As Imaging.ImageFormat
+                        Dim bDP As New ErrorsDescriber(EDP.None)
+                        Dim MD5BS As Func(Of String, UTypes,
+                                             SFile, Boolean, String) = Function(ByVal __URL As String, ByVal __MT As UTypes,
+                                                                                ByVal __File As SFile, ByVal __IsBase As Boolean) As String
+                                                                           Try
+                                                                               If __MT = UTypes.GIF Then
+                                                                                   ImgFormat = Imaging.ImageFormat.Gif
+                                                                               ElseIf __IsBase Then
+                                                                                   ImgFormat = GetImageFormat(CStr(RegexReplace(__URL, UrlBasePattern)))
+                                                                               Else
+                                                                                   ImgFormat = GetImageFormat(__File)
+                                                                               End If
+                                                                               Return ByteArrayToString(GetMD5(SFile.GetBytesFromNet(__URL, bDP), ImgFormat))
+                                                                           Catch hash_ex As Exception
+                                                                               Return String.Empty
+                                                                           End Try
+                                                                       End Function
+                        Dim m$
+                        Using w As New WebClient
+                            If vsf Then SFileShares.SFileExists($"{MyDir}\Video\", SFO.Path)
+                            Progress.TotalCount += _ContentNew.Count
+                            For i = 0 To _ContentNew.Count - 1
+                                Token.ThrowIfCancellationRequested()
+                                v = _ContentNew(i)
+                                v.State = UStates.Tried
+                                If v.Type = UTypes.Picture Then v.File = v.URL
+                                If v.File.IsEmptyString Then
+                                    f = v.URL
+                                Else
+                                    f = v.File
+                                End If
+                                f.Separator = "\"
+                                m = String.Empty
+                                If (v.Type = UTypes.Picture Or v.Type = UTypes.GIF) And Not cached Then
+                                    m = MD5BS(v.URL, v.Type, f, False)
+                                    If m.IsEmptyString AndAlso Not v.URL_BASE.IsEmptyString AndAlso Not v.URL_BASE = v.URL Then
+                                        m = MD5BS(v.URL_BASE, v.Type, f, True)
+                                        If Not m.IsEmptyString Then v.URL = v.URL_BASE
+                                    End If
+                                End If
+
+                                If (Not m.IsEmptyString AndAlso Not HashList.Contains(m)) Or Not (v.Type = UTypes.Picture Or
+                                                                                                  v.Type = UTypes.GIF) Or cached Then
+                                    If Not cached Then HashList.Add(m)
+                                    v.MD5 = m
+                                    f.Path = MyDir
+                                    Try
+                                        If (v.Type = UTypes.Video Or v.Type = UTypes.m3u8) And vsf Then f.Path = $"{f.PathWithSeparator}Video"
+                                        If v.Type = UTypes.m3u8 Then
+                                            f = M3U8.Download(v.URL, f)
+                                        Else
+                                            w.DownloadFile(v.URL, f.ToString)
+                                        End If
+                                        If Not v.Type = UTypes.m3u8 Or Not f.IsEmptyString Then
+                                            Select Case v.Type
+                                                Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
+                                                Case UTypes.Video, UTypes.m3u8 : DownloadedVideos += 1 : _CountVideo += 1
+                                            End Select
+                                            v.File = f
+                                            v.Post.CachedFile = f
+                                            v.State = UStates.Downloaded
+                                        End If
+                                    Catch wex As Exception
+                                        If Not IsChannel Then ErrorDownloading(f, v.URL)
+                                    End Try
+                                Else
+                                    v.State = UStates.Skipped
+                                End If
+                                _ContentNew(i) = v
+                                Progress.Perform()
+                            Next
+                        End Using
+                    End If
+                End If
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested
+            Catch ex As Exception
+                LogError(ex, "content downloading error")
+                HasError = True
+            End Try
+        End Sub
+        Protected Function GetSiteResponse(ByVal URL As String) As String
+            Try
+                Return Settings.Site(Sites.Reddit).Responser.GetResponse(URL,, EDP.ThrowException)
+            Catch ex As Exception
+                HasError = True
+                Dim e As EDP = EDP.SendInLog
+                Dim OptText$ = String.Empty
+                If Settings.Site(Sites.Reddit).Responser.StatusCode = HttpStatusCode.NotFound Then
+                    e += EDP.ThrowException
+                    OptText = ": USER NOT FOUND"
+                Else
+                    e += EDP.ReturnValue
+                End If
+                Return ErrorsDescriber.Execute(e, ex, $"[{Site} - {Name}: GetSiteResponse([{URL}])]{OptText}", String.Empty)
+            End Try
+        End Function
+        Protected Overrides Sub Dispose(ByVal disposing As Boolean)
+            If Not disposedValue And disposing Then ChannelPostsNames.Clear() : _ExistsUsersNames.Clear()
+            MyBase.Dispose(disposing)
+        End Sub
+    End Class
+End Namespace