mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 00:02:17 +00:00
1.0.0.1
Added: limited downloads, remove active downloads, simulation tasks management, copy channel image, x86 Removed reparse function Fixed some errors, typo and Reddit parsing algo
This commit is contained in:
Andy
@@ -40,6 +40,7 @@ Namespace API.Base
|
||||
Private Const Name_ParseUserMediaOnly As String = "ParseUserMediaOnly"
|
||||
Private Const Name_Temporary As String = "Temporary"
|
||||
Private Const Name_Favorite As String = "Favorite"
|
||||
Private Const Name_CreatedByChannel As String = "CreatedByChannel"
|
||||
|
||||
Private Const Name_SeparateVideoFolder As String = "SeparateVideoFolder"
|
||||
Private Const Name_CollectionName As String = "Collection"
|
||||
@@ -103,6 +104,7 @@ Namespace API.Base
|
||||
Return User.IsChannel
|
||||
End Get
|
||||
End Property
|
||||
Friend Property CreatedByChannel As Boolean = False
|
||||
Friend ReadOnly Property Self As IUserData Implements IUserData.Self
|
||||
Get
|
||||
Return Me
|
||||
@@ -235,7 +237,6 @@ BlockNullPicture:
|
||||
#Region "Content"
|
||||
Protected ReadOnly _ContentList As List(Of UserMedia)
|
||||
Protected ReadOnly _ContentNew As List(Of UserMedia)
|
||||
Protected ReadOnly _ContentForReparse As List(Of UserMedia)
|
||||
Protected ReadOnly _TempMediaList As List(Of UserMedia)
|
||||
Protected ReadOnly _TempPostsList As List(Of String)
|
||||
#End Region
|
||||
@@ -250,8 +251,6 @@ BlockNullPicture:
|
||||
End Set
|
||||
End Property
|
||||
Protected MyFileData As SFile
|
||||
Protected MyFileDataR As SFile
|
||||
Protected MyFileDataRV As SFile
|
||||
Protected MyFilePosts As SFile
|
||||
Friend Overridable Property FileExists As Boolean = False Implements IUserData.FileExists
|
||||
Friend Overridable Property DataMerging As Boolean
|
||||
@@ -398,7 +397,6 @@ BlockNullPicture:
|
||||
_InvokeImageHandler = InvokeImageHandler
|
||||
_ContentList = New List(Of UserMedia)
|
||||
_ContentNew = New List(Of UserMedia)
|
||||
_ContentForReparse = New List(Of UserMedia)
|
||||
_TempMediaList = New List(Of UserMedia)
|
||||
_TempPostsList = New List(Of String)
|
||||
Labels = New List(Of String)
|
||||
@@ -430,6 +428,7 @@ BlockNullPicture:
|
||||
ParseUserMediaOnly = x.Value(Name_ParseUserMediaOnly).FromXML(Of Boolean)(False)
|
||||
Temporary = x.Value(Name_Temporary).FromXML(Of Boolean)(False)
|
||||
Favorite = x.Value(Name_Favorite).FromXML(Of Boolean)(False)
|
||||
CreatedByChannel = x.Value(Name_CreatedByChannel).FromXML(Of Boolean)(False)
|
||||
SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
|
||||
ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
|
||||
_CountVideo = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
|
||||
@@ -440,7 +439,6 @@ BlockNullPicture:
|
||||
Labels.ListAddList(x.Value(Name_LabelsName).StringToList(Of String, List(Of String))("|", EDP.ReturnValue), LAP.NotContainsOnly, LAP.ClearBeforeAdd)
|
||||
End Using
|
||||
UpdateDataFiles()
|
||||
_DataForReparseExists = MyFileDataR.Exists
|
||||
End If
|
||||
Catch ex As Exception
|
||||
LogError(ex, "user information loading error")
|
||||
@@ -458,6 +456,7 @@ BlockNullPicture:
|
||||
x.Add(Name_ParseUserMediaOnly, ParseUserMediaOnly.BoolToInteger)
|
||||
x.Add(Name_Temporary, Temporary.BoolToInteger)
|
||||
x.Add(Name_Favorite, Favorite.BoolToInteger)
|
||||
x.Add(Name_CreatedByChannel, CreatedByChannel.BoolToInteger)
|
||||
If SeparateVideoFolder.HasValue Then
|
||||
x.Add(Name_SeparateVideoFolder, SeparateVideoFolder.Value.BoolToInteger)
|
||||
Else
|
||||
@@ -483,9 +482,6 @@ BlockNullPicture:
|
||||
Friend Overridable Overloads Sub LoadContentInformation()
|
||||
UpdateDataFiles()
|
||||
LoadContentInformation(_ContentList, MyFileData)
|
||||
LoadContentInformation(_ContentForReparse, MyFileDataR)
|
||||
LoadContentInformation(_TempMediaList, MyFileDataRV)
|
||||
_DataForReparseExists = False
|
||||
End Sub
|
||||
Private Overloads Sub LoadContentInformation(ByRef _CLIST As List(Of UserMedia), ByVal f As SFile)
|
||||
Try
|
||||
@@ -570,13 +566,7 @@ BlockNullPicture:
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download functions and options"
|
||||
Friend Overridable Property DownloadReparseOnly As Boolean = False Implements IUserData.DownloadReparseOnly
|
||||
Private _DataForReparseExists As Boolean = False
|
||||
Friend Overridable ReadOnly Property DataForReparseExists As Boolean Implements IUserData.DataForReparseExists
|
||||
Get
|
||||
Return _ContentForReparse.Count > 0 Or _DataForReparseExists
|
||||
End Get
|
||||
End Property
|
||||
Friend Overridable Property DownloadTopCount As Integer? = Nothing Implements IUserData.DownloadTopCount
|
||||
Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData
|
||||
Dim Canceled As Boolean = False
|
||||
Try
|
||||
@@ -586,79 +576,60 @@ BlockNullPicture:
|
||||
_DownloadedVideosSession = 0
|
||||
_TempMediaList.Clear()
|
||||
_TempPostsList.Clear()
|
||||
Dim __SaveData As Boolean = Not CreatedByChannel Or Not Settings.FromChannelDownloadTopUse
|
||||
|
||||
If Not _DataLoaded Then LoadContentInformation()
|
||||
|
||||
If Not DownloadReparseOnly Then
|
||||
If MyFilePosts.Exists Then _TempPostsList.ListAddList(File.ReadAllLines(MyFilePosts))
|
||||
If _ContentList.Count > 0 Then _TempPostsList.ListAddList(_ContentList.Select(Function(u) u.Post.ID), LNC)
|
||||
Token.ThrowIfCancellationRequested()
|
||||
DownloadDataF(Token)
|
||||
Token.ThrowIfCancellationRequested()
|
||||
End If
|
||||
If MyFilePosts.Exists Then _TempPostsList.ListAddList(File.ReadAllLines(MyFilePosts))
|
||||
If _ContentList.Count > 0 Then _TempPostsList.ListAddList(_ContentList.Select(Function(u) u.Post.ID), LNC)
|
||||
ThrowAny(Token)
|
||||
DownloadDataF(Token)
|
||||
ThrowAny(Token)
|
||||
|
||||
ReparseVideo(Token)
|
||||
If Token.IsCancellationRequested Then
|
||||
If Not DownloadReparseOnly Then
|
||||
If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(c) c.Type = UserMedia.Types.VideoPre) Then
|
||||
TextSaver.SaveTextToFile((From c As UserMedia In _TempMediaList
|
||||
Where c.Type = UserMedia.Types.VideoPre
|
||||
Select c.URL).ListToString(, Environment.NewLine), MyFileDataRV, True,, EDP.SendInLog)
|
||||
Else
|
||||
If MyFileDataRV.Exists Then MyFileDataRV.Delete(,,, EDP.SendInLog)
|
||||
End If
|
||||
End If
|
||||
Else
|
||||
If Not DownloadReparseOnly And _TempPostsList.Count > 0 Then TextSaver.SaveTextToFile(_TempPostsList.ListToString(, Environment.NewLine), MyFilePosts, True,, EDP.None)
|
||||
_ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
|
||||
End If
|
||||
|
||||
Dim r% = 0
|
||||
Do While r <= 2 And (_ContentNew.Count > 0 Or _ContentForReparse.Count > 0) And Not Token.IsCancellationRequested
|
||||
DownloadContent(Token)
|
||||
If _ContentNew.Count > 0 Then
|
||||
_ContentForReparse.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Tried Or c.State = UState.Unknown), LNC)
|
||||
_ContentList.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Downloaded), LNC)
|
||||
End If
|
||||
If _ContentForReparse.Count > 0 Then _ContentForReparse.RemoveAll(Function(c) _ContentList.Contains(c))
|
||||
_ContentNew.Clear()
|
||||
If _ContentForReparse.Count > 0 Then _ContentNew.ListAddList(_ContentForReparse, LNC)
|
||||
r += 1
|
||||
Loop
|
||||
|
||||
ThrowAny(Token)
|
||||
If _TempPostsList.Count > 0 And __SaveData Then TextSaver.SaveTextToFile(_TempPostsList.ListToString(, Environment.NewLine), MyFilePosts, True,, EDP.None)
|
||||
_ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
|
||||
DownloadContent(Token)
|
||||
ThrowIfDisposed()
|
||||
_ContentList.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Downloaded), LNC)
|
||||
_CountPictures = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Picture)
|
||||
_CountVideo = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Video)
|
||||
If DownloadedPictures + DownloadedVideos > 0 Then
|
||||
LastUpdated = Now
|
||||
If Labels.Contains(LabelsKeeper.NoParsedUser) Then Labels.Remove(LabelsKeeper.NoParsedUser)
|
||||
UpdateContentInformation(_ContentList, MyFileData)
|
||||
If __SaveData Then
|
||||
LastUpdated = Now
|
||||
If Labels.Contains(LabelsKeeper.NoParsedUser) Then Labels.Remove(LabelsKeeper.NoParsedUser)
|
||||
UpdateContentInformation(_ContentList, MyFileData)
|
||||
Else
|
||||
_CountVideo = 0
|
||||
_CountPictures = 0
|
||||
_ContentList.Clear()
|
||||
CreatedByChannel = False
|
||||
End If
|
||||
UpdateUserInformation()
|
||||
End If
|
||||
If _ContentForReparse.Count > 0 Then UpdateContentInformation(_ContentForReparse, MyFileDataR)
|
||||
ThrowIfDisposed()
|
||||
_DownloadedPicturesTotal += _DownloadedPicturesSession
|
||||
_DownloadedVideosTotal += _DownloadedVideosSession
|
||||
If UpPic Then Raise_OnPictureUpdated()
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
MyMainLOG = $"{Site} - {Name}: downloading canceled"
|
||||
Canceled = True
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Canceled = True
|
||||
Catch ex As Exception
|
||||
LogError(ex, "downloading data error")
|
||||
HasError = True
|
||||
Finally
|
||||
If Not Canceled Then _DataParsed = True ': LastUpdated = Now
|
||||
_ContentNew.Clear()
|
||||
DownloadReparseOnly = False
|
||||
If _ContentForReparse.Count = 0 And MyFileDataR.Exists Then MyFileDataR.Delete(,,, EDP.SendInLog)
|
||||
DownloadTopCount = Nothing
|
||||
End Try
|
||||
End Sub
|
||||
Private Sub UpdateDataFiles()
|
||||
If Not User.File.IsEmptyString Then
|
||||
MyFileData = User.File
|
||||
MyFileData.Name &= "_Data"
|
||||
MyFileDataR = MyFileData
|
||||
MyFileDataR.Name &= "_REPARSE"
|
||||
MyFileDataRV = MyFileData
|
||||
MyFileDataRV.Name &= "_RVideo"
|
||||
MyFilePosts = User.File
|
||||
MyFilePosts.Name &= "_Posts"
|
||||
MyFilePosts.Extension = "txt"
|
||||
@@ -687,9 +658,10 @@ BlockNullPicture:
|
||||
End Function
|
||||
Friend Overridable Function MoveFiles(ByVal __CollectionName As String, ByVal _MergeData As Boolean) As Boolean Implements IUserData.MoveFiles
|
||||
Dim UserBefore As UserInfo = User
|
||||
Dim Removed As Boolean = True
|
||||
Dim _TurnBack As Boolean = False
|
||||
Try
|
||||
Dim f As SFile
|
||||
Dim Removed As Boolean
|
||||
If IncludedInCollection Then
|
||||
Settings.Users.Add(Me)
|
||||
Removed = False
|
||||
@@ -701,6 +673,7 @@ BlockNullPicture:
|
||||
User.CollectionName = __CollectionName
|
||||
User.IncludedInCollection = True
|
||||
End If
|
||||
_TurnBack = True
|
||||
User.UpdateUserFile()
|
||||
f = User.File.CutPath(, EDP.ThrowException)
|
||||
If f.Exists(SFO.Path, False) Then
|
||||
@@ -711,6 +684,7 @@ BlockNullPicture:
|
||||
MsgBoxE("Operation canceled", MsgBoxStyle.Exclamation)
|
||||
User = UserBefore
|
||||
If Removed Then Settings.Users.Add(Me) Else Settings.Users.Remove(Me)
|
||||
_TurnBack = False
|
||||
Return False
|
||||
End If
|
||||
f.Delete(SFO.Path, False, False, EDP.ThrowException)
|
||||
@@ -724,6 +698,9 @@ BlockNullPicture:
|
||||
Catch ex As Exception
|
||||
ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Files moving error")
|
||||
User = UserBefore
|
||||
If _TurnBack Then
|
||||
If Removed Then Settings.Users.Add(Me) Else Settings.Users.Remove(Me)
|
||||
End If
|
||||
Return False
|
||||
End Try
|
||||
End Function
|
||||
@@ -789,6 +766,16 @@ BlockNullPicture:
|
||||
Protected Sub ErrorDownloading(ByVal f As SFile, ByVal URL As String)
|
||||
If Not f.Exists Then MyMainLOG = $"Error downloading from [{URL}] to [{f}]"
|
||||
End Sub
|
||||
''' <exception cref="ObjectDisposedException"></exception>
|
||||
Protected Sub ThrowIfDisposed()
|
||||
If Disposed Then Throw New ObjectDisposedException(ToString(), "Object disposed")
|
||||
End Sub
|
||||
''' <exception cref="OperationCanceledException"></exception>
|
||||
''' <exception cref="ObjectDisposedException"></exception>
|
||||
Protected Sub ThrowAny(ByVal Token As CancellationToken)
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowIfDisposed()
|
||||
End Sub
|
||||
#End Region
|
||||
Public Overrides Function ToString() As String
|
||||
If Settings.ViewModeIsPicture Then
|
||||
@@ -861,7 +848,7 @@ BlockNullPicture:
|
||||
#End Region
|
||||
#Region "IDisposable Support"
|
||||
Protected disposedValue As Boolean = False
|
||||
Friend ReadOnly Property Disposed As Boolean
|
||||
Friend ReadOnly Property Disposed As Boolean Implements IUserData.Disposed
|
||||
Get
|
||||
Return disposedValue
|
||||
End Get
|
||||
@@ -942,9 +929,9 @@ BlockNullPicture:
|
||||
Function Delete() As Integer
|
||||
Function MoveFiles(ByVal CollectionName As String, ByVal MergeData As Boolean) As Boolean
|
||||
Sub OpenFolder()
|
||||
Property DownloadReparseOnly As Boolean
|
||||
ReadOnly Property DataForReparseExists As Boolean
|
||||
ReadOnly Property Self As IUserData
|
||||
Property DownloadTopCount As Integer?
|
||||
ReadOnly Property Disposed As Boolean
|
||||
End Interface
|
||||
Friend Interface IChannelLimits
|
||||
Property AutoGetLimits As Boolean
|
||||
|
||||
@@ -222,7 +222,7 @@ Namespace API.Reddit
|
||||
End If
|
||||
End Function
|
||||
#End Region
|
||||
#Region "IXMLContainer Support"
|
||||
#Region "ILoaderSaver Support"
|
||||
Friend Overloads Function LoadData(Optional ByVal f As SFile = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Implements ILoaderSaver.Load
|
||||
Return LoadData(File, False, e)
|
||||
End Function
|
||||
|
||||
@@ -72,16 +72,18 @@ Namespace API.Reddit
|
||||
Friend Sub DownloadData(ByVal Token As CancellationToken, Optional ByVal SkipExists As Boolean = True,
|
||||
Optional ByVal p As MyProgress = Nothing)
|
||||
Try
|
||||
Dim m% = Settings.ChannelsMaxJobsCount
|
||||
If Count > 0 Then
|
||||
Dim t As New List(Of Task)
|
||||
Dim i% = 0
|
||||
For Each c As Channel In Channels
|
||||
If Not c.Downloading Then t.Add(Task.Run(Sub()
|
||||
c.SetLimit(Me)
|
||||
c.DownloadData(Token, SkipExists, p)
|
||||
End Sub))
|
||||
End Sub)) : i += 1
|
||||
If t.Count > 0 And i >= m Then Task.WaitAll(t.ToArray, Token) : t.Clear() : i = 0
|
||||
Next
|
||||
If t.Count > 0 Then Task.WaitAll(t.ToArray)
|
||||
Token.ThrowIfCancellationRequested()
|
||||
If t.Count > 0 Then Task.WaitAll(t.ToArray, Token) : t.Clear()
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
End Try
|
||||
|
||||
@@ -80,7 +80,7 @@ Namespace API.Reddit
|
||||
eFiles.Add(dFile)
|
||||
Next
|
||||
End Using
|
||||
f = FFMPEG.ConcatenateFiles(eFiles, "ffmpeg.exe", ConcatFile, p, DPED)
|
||||
f = FFMPEG.ConcatenateFiles(eFiles, Settings.FfmpegFile, ConcatFile, p, DPED)
|
||||
eFiles.Clear()
|
||||
Return f
|
||||
End If
|
||||
|
||||
@@ -78,8 +78,8 @@ Namespace API.Reddit
|
||||
End If
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
_TotalPostsDownloaded = 0
|
||||
If IsChannel Then
|
||||
_DownloadedChannelPosts = 0
|
||||
DownloadDataChannel(String.Empty, Token)
|
||||
Else
|
||||
DownloadDataUser(String.Empty, Token)
|
||||
@@ -87,6 +87,7 @@ Namespace API.Reddit
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download Functions (User, Channel)"
|
||||
Private _TotalPostsDownloaded As Integer = 0
|
||||
Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
@@ -103,7 +104,7 @@ Namespace API.Reddit
|
||||
Dim UPicType As Func(Of String, UTypes) = Function(input) IIf(input = "image", UTypes.Picture, UTypes.GIF)
|
||||
|
||||
URL = $"https://gateway.reddit.com/desktopapi/v1/user/{Name}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
Dim r$ = GetSiteResponse(URL)
|
||||
If Not r.IsEmptyString Then
|
||||
Using w As EContainer = JsonDocument.Parse(r)
|
||||
@@ -111,7 +112,7 @@ Namespace API.Reddit
|
||||
n = w.GetNode(JsonNodesJson)
|
||||
If Not n Is Nothing AndAlso n.Count > 0 Then
|
||||
For Each nn In n
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
If nn.Count > 0 Then
|
||||
PostID = nn.Name
|
||||
If PostID.IsEmptyString AndAlso nn.Contains("id") Then PostID = nn("id").Value
|
||||
@@ -129,19 +130,24 @@ Namespace API.Reddit
|
||||
s = nn.ItemF({"source", "url"})
|
||||
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
Else
|
||||
s = nn.ItemF({"media"}).XmlIfNothing
|
||||
__ItemType = s("type").XmlIfNothingValue
|
||||
Select Case __ItemType
|
||||
Case "gallery" : DownloadGallery(s, PostID, PostDate)
|
||||
Case "gallery" : DownloadGallery(s, PostID, PostDate) : _TotalPostsDownloaded += 1
|
||||
Case "image", "gifvideo"
|
||||
If s.Contains("content") Then _
|
||||
If s.Contains("content") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
|
||||
PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
Case "video"
|
||||
If s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then _
|
||||
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
|
||||
PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
Case Else : added = False
|
||||
End Select
|
||||
End If
|
||||
@@ -151,14 +157,16 @@ Namespace API.Reddit
|
||||
With s.Value.ToLower
|
||||
Select Case True
|
||||
Case .Contains("redgifs") : tmpType = UTypes.VideoPre
|
||||
Case .Contains("m3u8") : tmpType = UTypes.m3u8
|
||||
Case .Contains("m3u8") : If Settings.UseM3U8 Then tmpType = UTypes.m3u8
|
||||
Case .Contains(".gif") And TryFile(s.Value) : tmpType = UTypes.GIF
|
||||
Case TryFile(s.Value) : tmpType = UTypes.Picture
|
||||
Case Else : tmpType = UTypes.Undefined
|
||||
End Select
|
||||
End With
|
||||
If Not tmpType = UTypes.Undefined Then _
|
||||
If Not tmpType = UTypes.Undefined Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
@@ -171,12 +179,12 @@ Namespace API.Reddit
|
||||
If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataUser(PostID, Token)
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
LogError(ex, $"data downloading error [{URL}]")
|
||||
HasError = True
|
||||
End Try
|
||||
End Sub
|
||||
Private _DownloadedChannelPosts As Integer = 0
|
||||
Private Sub DownloadDataChannel(ByVal POST As String, ByVal Token As CancellationToken)
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
@@ -189,7 +197,7 @@ Namespace API.Reddit
|
||||
Dim lDate As Date?
|
||||
|
||||
URL = $"https://reddit.com/r/{Name}/new.json?allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
Dim r$ = GetSiteResponse(URL)
|
||||
If Not r.IsEmptyString Then
|
||||
Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
|
||||
@@ -197,14 +205,14 @@ Namespace API.Reddit
|
||||
n = w.GetNode(ChannelJsonNodes)
|
||||
If Not n Is Nothing AndAlso n.Count > 0 Then
|
||||
For Each nn In n
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
s = nn.ItemF({eCount})
|
||||
If Not s Is Nothing AndAlso s.Count > 0 Then
|
||||
PostID = s.Value("name")
|
||||
If PostID.IsEmptyString AndAlso s.Contains("id") Then PostID = s("id").Value
|
||||
|
||||
If ChannelPostsNames.Contains(PostID) Then ExistsDetected = True : Continue For 'Exit Sub
|
||||
If DownloadLimitCount.HasValue AndAlso _DownloadedChannelPosts >= DownloadLimitCount.Value Then Exit Sub
|
||||
If DownloadLimitCount.HasValue AndAlso _TotalPostsDownloaded >= DownloadLimitCount.Value Then Exit Sub
|
||||
If Not DownloadLimitPost.IsEmptyString AndAlso DownloadLimitPost = PostID Then Exit Sub
|
||||
If DownloadLimitDate.HasValue AndAlso _TempMediaList.Count > 0 Then
|
||||
With (From __u In _TempMediaList Where __u.Post.Date.HasValue Select __u.Post.Date.Value)
|
||||
@@ -226,20 +234,20 @@ Namespace API.Reddit
|
||||
tmpUrl = s.Value({"media", "oembed"}, "thumbnail_url")
|
||||
If Not tmpUrl.IsEmptyString Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_DownloadedChannelPosts += 1
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
Else
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, tmpUrl, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_DownloadedChannelPosts += 1
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
ElseIf s.Item("media_metadata").XmlIfNothing.Count > 0 Then
|
||||
DownloadGallery(s, PostID, PostDate, _UserID, SaveToCache)
|
||||
_DownloadedChannelPosts += 1
|
||||
_TotalPostsDownloaded += 1
|
||||
ElseIf s.Contains("preview") Then
|
||||
ss = s.ItemF({"preview", "images", eCount, "source", "url"}).XmlIfNothing
|
||||
If Not ss.Value.IsEmptyString Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, ss.Value, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_DownloadedChannelPosts += 1
|
||||
_TotalPostsDownloaded += 1
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
@@ -251,6 +259,7 @@ Namespace API.Reddit
|
||||
If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataChannel(PostID, Token)
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
LogError(ex, $"channel data downloading error [{URL}]")
|
||||
HasError = True
|
||||
@@ -279,15 +288,16 @@ Namespace API.Reddit
|
||||
End Sub
|
||||
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(p) p.Type = UTypes.VideoPre) Then
|
||||
Dim r$, v$
|
||||
Dim e As New ErrorsDescriber(EDP.ReturnValue)
|
||||
Dim m As UserMedia
|
||||
For i% = _TempMediaList.Count - 1 To 0 Step -1
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
If _TempMediaList(i).Type = UTypes.VideoPre Then
|
||||
m = _TempMediaList(i)
|
||||
r = GetSiteResponse(m.URL)
|
||||
r = GetSiteResponse(m.URL, e)
|
||||
_TempMediaList(i) = New UserMedia
|
||||
If Not r.IsEmptyString Then
|
||||
v = RegexReplace(r, VideoRegEx)
|
||||
@@ -301,6 +311,7 @@ Namespace API.Reddit
|
||||
Next
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
LogError(ex, "video reparsing error")
|
||||
End Try
|
||||
@@ -326,7 +337,7 @@ Namespace API.Reddit
|
||||
If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing
|
||||
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
|
||||
Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
|
||||
If t = UTypes.Picture Or t = UTypes.GIF Then m.File = CStr(RegexReplace(m.URL, FilesPattern)) Else m.File = Nothing
|
||||
If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing
|
||||
If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
|
||||
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
|
||||
Return m
|
||||
@@ -342,11 +353,15 @@ Namespace API.Reddit
|
||||
Return False
|
||||
End Try
|
||||
End Function
|
||||
Private Shared Function UrlToFile(ByVal URL As String) As SFile
|
||||
Return CStr(RegexReplace(URL, FilesPattern))
|
||||
End Function
|
||||
#End Region
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Dim i%
|
||||
Token.ThrowIfCancellationRequested()
|
||||
Dim dCount% = 0, dTotal% = 0
|
||||
ThrowAny(Token)
|
||||
If _ContentNew.Count > 0 Then
|
||||
_ContentNew.RemoveAll(Function(c) c.URL.IsEmptyString)
|
||||
If _ContentNew.Count > 0 Then
|
||||
@@ -386,12 +401,11 @@ Namespace API.Reddit
|
||||
If vsf Then SFileShares.SFileExists($"{MyDir}\Video\", SFO.Path)
|
||||
Progress.TotalCount += _ContentNew.Count
|
||||
For i = 0 To _ContentNew.Count - 1
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
v = _ContentNew(i)
|
||||
v.State = UStates.Tried
|
||||
If v.Type = UTypes.Picture Then v.File = v.URL
|
||||
If v.File.IsEmptyString Then
|
||||
f = v.URL
|
||||
f = UrlToFile(v.URL)
|
||||
Else
|
||||
f = v.File
|
||||
End If
|
||||
@@ -425,6 +439,7 @@ Namespace API.Reddit
|
||||
v.File = f
|
||||
v.Post.CachedFile = f
|
||||
v.State = UStates.Downloaded
|
||||
dCount += 1
|
||||
End If
|
||||
Catch wex As Exception
|
||||
If Not IsChannel Then ErrorDownloading(f, v.URL)
|
||||
@@ -433,29 +448,40 @@ Namespace API.Reddit
|
||||
v.State = UStates.Skipped
|
||||
End If
|
||||
_ContentNew(i) = v
|
||||
Progress.Perform()
|
||||
If (CreatedByChannel And Settings.FromChannelDownloadTopUse And dCount >= Settings.FromChannelDownloadTop) Or
|
||||
(DownloadTopCount.HasValue AndAlso dCount >= DownloadTopCount.Value) Then
|
||||
Progress.Perform(_ContentNew.Count - dTotal)
|
||||
Exit Sub
|
||||
Else
|
||||
dTotal += 1
|
||||
Progress.Perform()
|
||||
End If
|
||||
Next
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
LogError(ex, "content downloading error")
|
||||
HasError = True
|
||||
End Try
|
||||
End Sub
|
||||
Protected Function GetSiteResponse(ByVal URL As String) As String
|
||||
Protected Function GetSiteResponse(ByVal URL As String, Optional ByVal e As ErrorsDescriber = Nothing) As String
|
||||
Try
|
||||
Return Settings.Site(Sites.Reddit).Responser.GetResponse(URL,, EDP.ThrowException)
|
||||
Catch ex As Exception
|
||||
HasError = True
|
||||
Dim e As EDP = EDP.SendInLog
|
||||
Dim OptText$ = String.Empty
|
||||
If Settings.Site(Sites.Reddit).Responser.StatusCode = HttpStatusCode.NotFound Then
|
||||
e += EDP.ThrowException
|
||||
OptText = ": USER NOT FOUND"
|
||||
Else
|
||||
e += EDP.ReturnValue
|
||||
If Not e.Exists Then
|
||||
Dim ee As EDP = EDP.SendInLog
|
||||
If Settings.Site(Sites.Reddit).Responser.StatusCode = HttpStatusCode.NotFound Then
|
||||
ee += EDP.ThrowException
|
||||
OptText = ": USER NOT FOUND"
|
||||
Else
|
||||
ee += EDP.ReturnValue
|
||||
End If
|
||||
e = New ErrorsDescriber(ee)
|
||||
End If
|
||||
Return ErrorsDescriber.Execute(e, ex, $"[{Site} - {Name}: GetSiteResponse([{URL}])]{OptText}", String.Empty)
|
||||
End Try
|
||||
|
||||
@@ -58,13 +58,13 @@ Namespace API.Twitter
|
||||
URL = $"https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={Name}&count=200&exclude_replies=false&include_rts=1&tweet_mode=extended"
|
||||
If Not POST.IsEmptyString Then URL &= $"&max_id={POST}"
|
||||
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
Dim r$ = Settings.Site(Sites.Twitter).Responser.GetResponse(URL,, EDP.ThrowException)
|
||||
If Not r.IsEmptyString Then
|
||||
Using w As EContainer = JsonDocument.Parse(r)
|
||||
If Not w Is Nothing AndAlso w.Count > 0 Then
|
||||
For Each nn In w
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
If nn.Count > 0 Then
|
||||
PostID = nn.Value("id")
|
||||
If ID.IsEmptyString Then
|
||||
@@ -107,6 +107,7 @@ Namespace API.Twitter
|
||||
If Not PostID.IsEmptyString And NewPostDetected Then DownloadData(PostID, Token)
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
LogError(ex, $"data downloading error [{URL}]")
|
||||
HasError = True
|
||||
@@ -206,7 +207,7 @@ Namespace API.Twitter
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Dim i%
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
If _ContentNew.Count > 0 Then
|
||||
_ContentNew.RemoveAll(Function(c) c.URL.IsEmptyString)
|
||||
If _ContentNew.Count > 0 Then
|
||||
@@ -219,7 +220,7 @@ Namespace API.Twitter
|
||||
If vsf Then SFileShares.SFileExists($"{MyDir}\Video\", SFO.Path)
|
||||
MainProgress.TotalCount += _ContentNew.Count
|
||||
For i = 0 To _ContentNew.Count - 1
|
||||
Token.ThrowIfCancellationRequested()
|
||||
ThrowAny(Token)
|
||||
v = _ContentNew(i)
|
||||
v.State = UStates.Tried
|
||||
If v.File.IsEmptyString Then
|
||||
@@ -255,6 +256,7 @@ Namespace API.Twitter
|
||||
End If
|
||||
End If
|
||||
Catch oex As OperationCanceledException When Token.IsCancellationRequested
|
||||
Catch dex As ObjectDisposedException When Disposed
|
||||
Catch ex As Exception
|
||||
LogError(ex, "content downloading error")
|
||||
HasError = True
|
||||
|
||||
@@ -20,6 +20,14 @@ Namespace API
|
||||
ChangeCollectionName(NewName, True)
|
||||
End Set
|
||||
End Property
|
||||
Friend Overrides Property Name As String
|
||||
Get
|
||||
Return CollectionName
|
||||
End Get
|
||||
Set(ByVal NewCollectionName As String)
|
||||
CollectionName = NewCollectionName
|
||||
End Set
|
||||
End Property
|
||||
Friend Overrides Sub ChangeCollectionName(ByVal NewName As String, ByVal UpdateSettings As Boolean)
|
||||
_CollectionName = NewName
|
||||
If Count > 0 Then Collections.ForEach(Sub(c) c.CollectionName = NewName)
|
||||
@@ -233,22 +241,17 @@ Namespace API
|
||||
Friend Overrides Sub LoadContentInformation()
|
||||
If Count > 0 Then Collections.ForEach(Sub(c) DirectCast(c, UserDataBase).LoadContentInformation())
|
||||
End Sub
|
||||
Friend Overrides Property DownloadReparseOnly As Boolean
|
||||
Get
|
||||
If Count > 0 Then Return Collections(0).DownloadReparseOnly Else Return False
|
||||
End Get
|
||||
Set(ByVal DRO As Boolean)
|
||||
If Count > 0 Then Collections.ForEach(Sub(u) u.DownloadReparseOnly = DRO)
|
||||
End Set
|
||||
End Property
|
||||
Friend Overrides ReadOnly Property DataForReparseExists As Boolean
|
||||
Friend Overrides Property DownloadTopCount As Integer?
|
||||
Get
|
||||
If Count > 0 Then
|
||||
Return Collections.Exists(Function(u) u.DataForReparseExists)
|
||||
Return Collections(0).DownloadTopCount
|
||||
Else
|
||||
Return False
|
||||
Return Nothing
|
||||
End If
|
||||
End Get
|
||||
Set(ByVal NewLimit As Integer?)
|
||||
If Count > 0 Then Collections.ForEach(Sub(c) c.DownloadTopCount = NewLimit)
|
||||
End Set
|
||||
End Property
|
||||
Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
|
||||
If Count > 0 Then Downloader.AddRange(Collections)
|
||||
@@ -282,16 +285,24 @@ Namespace API
|
||||
Return False
|
||||
End Get
|
||||
End Property
|
||||
''' <exception cref="InvalidOperationException"></exception>
|
||||
Friend Overloads Sub Add(ByVal _Item As IUserData) Implements ICollection(Of IUserData).Add
|
||||
With _Item
|
||||
.Temporary = Temporary
|
||||
.Favorite = Favorite
|
||||
ImageHandler(_Item, False)
|
||||
AddHandler _Item.OnPictureUpdated, AddressOf User_OnPictureUpdated
|
||||
Dim m As Boolean = DataMerging
|
||||
If .MoveFiles(CollectionName, m) Then
|
||||
Collections.Add(_Item)
|
||||
DirectCast(_Item, UserDataBase).CreateButtons(Count - 1)
|
||||
With Collections.Last
|
||||
If Collections.Count - 1 > 0 Then
|
||||
.Temporary = Temporary
|
||||
.Favorite = Favorite
|
||||
.UpdateUserInformation()
|
||||
End If
|
||||
ImageHandler(_Item, False)
|
||||
AddHandler .OnPictureUpdated, AddressOf User_OnPictureUpdated
|
||||
DirectCast(.Self, UserDataBase).CreateButtons(Count - 1)
|
||||
End With
|
||||
Else
|
||||
Throw New InvalidOperationException("User data doe not move to the collection folder")
|
||||
End If
|
||||
End With
|
||||
End Sub
|
||||
|
||||
Reference in New Issue
Block a user