mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-14 15:52:18 +00:00
1.0.0.2
Add: downloading certain types of media, name files by date Fix: Stop button is not activated after download start
This commit is contained in:
Andy
@@ -49,6 +49,7 @@
|
||||
''' <summary>Post ID</summary>
|
||||
Friend ID As String
|
||||
Friend [Date] As Date?
|
||||
Friend Title As String
|
||||
#Region "Channel compatible fields"
|
||||
Friend UserID As String
|
||||
Friend CachedFile As SFile
|
||||
|
||||
@@ -47,6 +47,8 @@ Namespace API.Base
|
||||
Private Const Name_LabelsName As String = "Labels"
|
||||
|
||||
Private Const Name_ReadyForDownload As String = "ReadyForDownload"
|
||||
Private Const Name_DownloadImages As String = "DownloadImages"
|
||||
Private Const Name_DownloadVideos As String = "DownloadVideos"
|
||||
|
||||
Private Const Name_VideoCount As String = "VideoCount"
|
||||
Private Const Name_PicturesCount As String = "PicturesCount"
|
||||
@@ -233,6 +235,8 @@ BlockNullPicture:
|
||||
Protected _DataLoaded As Boolean = False
|
||||
Protected _DataParsed As Boolean = False
|
||||
Friend Property ReadyForDownload As Boolean = True Implements IUserData.ReadyForDownload
|
||||
Friend Property DownloadImages As Boolean = True Implements IUserData.DownloadImages
|
||||
Friend Property DownloadVideos As Boolean = True Implements IUserData.DownloadVideos
|
||||
#End Region
|
||||
#Region "Content"
|
||||
Protected ReadOnly _ContentList As List(Of UserMedia)
|
||||
@@ -431,6 +435,8 @@ BlockNullPicture:
|
||||
CreatedByChannel = x.Value(Name_CreatedByChannel).FromXML(Of Boolean)(False)
|
||||
SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
|
||||
ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
|
||||
DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
|
||||
DownloadedVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
|
||||
_CountVideo = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
|
||||
_CountPictures = x.Value(Name_PicturesCount).FromXML(Of Integer)(0)
|
||||
LastUpdated = AConvert(Of Date)(x.Value(Name_LastUpdated), ADateTime.Formats.BaseDateTime, Nothing)
|
||||
@@ -463,6 +469,8 @@ BlockNullPicture:
|
||||
x.Add(Name_SeparateVideoFolder, String.Empty)
|
||||
End If
|
||||
x.Add(Name_ReadyForDownload, ReadyForDownload.BoolToInteger)
|
||||
x.Add(Name_DownloadImages, DownloadImages.BoolToInteger)
|
||||
x.Add(Name_DownloadVideos, DownloadVideos.BoolToInteger)
|
||||
x.Add(Name_VideoCount, _CountVideo)
|
||||
x.Add(Name_PicturesCount, _CountPictures)
|
||||
x.Add(Name_LastUpdated, AConvert(Of String)(LastUpdated, ADateTime.Formats.BaseDateTime, String.Empty))
|
||||
@@ -586,6 +594,12 @@ BlockNullPicture:
|
||||
DownloadDataF(Token)
|
||||
ThrowAny(Token)
|
||||
|
||||
If _TempMediaList.Count > 0 Then
|
||||
If Not DownloadImages Then _TempMediaList.RemoveAll(Function(m) m.Type = UserMedia.Types.GIF Or m.Type = UserMedia.Types.Picture)
|
||||
If Not DownloadVideos Then _TempMediaList.RemoveAll(Function(m) m.Type = UserMedia.Types.Video Or
|
||||
m.Type = UserMedia.Types.VideoPre Or m.Type = UserMedia.Types.m3u8)
|
||||
End If
|
||||
|
||||
ReparseVideo(Token)
|
||||
ThrowAny(Token)
|
||||
If _TempPostsList.Count > 0 And __SaveData Then TextSaver.SaveTextToFile(_TempPostsList.ListToString(, Environment.NewLine), MyFilePosts, True,, EDP.None)
|
||||
@@ -640,6 +654,29 @@ BlockNullPicture:
|
||||
Protected MustOverride Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
Protected MustOverride Sub ReparseVideo(ByVal Token As CancellationToken)
|
||||
Protected MustOverride Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
Protected Function ChangeFileNameByProvider(ByVal f As SFile, ByVal m As UserMedia) As SFile
|
||||
Dim ff As SFile = Nothing
|
||||
Try
|
||||
If Not f.IsEmptyString AndAlso f.Exists Then
|
||||
Dim d As Date? = m.Post.Date
|
||||
If Settings.FileReplaceNameByDate Then
|
||||
Dim dd$ = AConvert(Of String)(If(d, Now), FileDateAppenderProvider, String.Empty)
|
||||
ff = f
|
||||
ff.Name = dd
|
||||
ff = SFile.Indexed_IndexFile(ff,, New NumberedFile(ff))
|
||||
ElseIf d.HasValue AndAlso (Settings.FileAddDateToFileName Or Settings.FileAddTimeToFileName) AndAlso
|
||||
(Not FileDateAppenderProvider Is Nothing And Not FileDateAppenderPattern.IsEmptyString) Then
|
||||
ff = f
|
||||
ff.Name = String.Format(FileDateAppenderPattern, f.Name, CStr(AConvert(Of String)(d.Value, FileDateAppenderProvider, String.Empty)))
|
||||
End If
|
||||
If Not ff.Name.IsEmptyString Then My.Computer.FileSystem.RenameFile(f, ff.File) : Return ff
|
||||
End If
|
||||
Return f
|
||||
Catch ex As Exception
|
||||
LogError(ex, $"change file name from [{f}] to [{ff}]")
|
||||
Return f
|
||||
End Try
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Delete, Move, Merge"
|
||||
Friend Overridable Function Delete() As Integer Implements IUserData.Delete
|
||||
@@ -916,6 +953,8 @@ BlockNullPicture:
|
||||
ReadOnly Property FitToAddParams As Boolean
|
||||
ReadOnly Property LVIKey As String
|
||||
ReadOnly Property LVIIndex As Integer
|
||||
Property DownloadImages As Boolean
|
||||
Property DownloadVideos As Boolean
|
||||
Function GetLVI(ByVal Destination As ListView) As ListViewItem
|
||||
Function GetLVIGroup(ByVal Destination As ListView) As ListViewGroup
|
||||
Sub LoadUserInformation()
|
||||
|
||||
@@ -92,7 +92,7 @@ Namespace API.Reddit
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
Dim PostID$ = String.Empty
|
||||
Dim PostDate$
|
||||
Dim PostDate$, PostTitle$
|
||||
Dim n As EContainer, nn As EContainer, s As EContainer
|
||||
Dim NewPostDetected As Boolean = False
|
||||
Dim ExistsDetected As Boolean = False
|
||||
@@ -107,7 +107,7 @@ Namespace API.Reddit
|
||||
ThrowAny(Token)
|
||||
Dim r$ = GetSiteResponse(URL)
|
||||
If Not r.IsEmptyString Then
|
||||
Using w As EContainer = JsonDocument.Parse(r)
|
||||
Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
|
||||
If w.Count > 0 Then
|
||||
n = w.GetNode(JsonNodesJson)
|
||||
If Not n Is Nothing AndAlso n.Count > 0 Then
|
||||
@@ -124,29 +124,32 @@ Namespace API.Reddit
|
||||
ExistsDetected = True
|
||||
Continue For
|
||||
End If
|
||||
PostTitle = nn.Value("title")
|
||||
|
||||
If CheckNode(nn) Then
|
||||
_ItemsBefore = _TempMediaList.Count
|
||||
added = True
|
||||
s = nn.ItemF({"source", "url"})
|
||||
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
Else
|
||||
s = nn.ItemF({"media"}).XmlIfNothing
|
||||
__ItemType = s("type").XmlIfNothingValue
|
||||
Select Case __ItemType
|
||||
Case "gallery" : DownloadGallery(s, PostID, PostDate) : _TotalPostsDownloaded += 1
|
||||
Case "gallery" : If Not DownloadGallery(s, PostID, PostDate,,, PostTitle) Then added = False
|
||||
Case "image", "gifvideo"
|
||||
If s.Contains("content") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
|
||||
PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
Else
|
||||
added = False
|
||||
End If
|
||||
Case "video"
|
||||
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
|
||||
PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
Else
|
||||
added = False
|
||||
End If
|
||||
Case Else : added = False
|
||||
End Select
|
||||
@@ -164,8 +167,7 @@ Namespace API.Reddit
|
||||
End Select
|
||||
End With
|
||||
If Not tmpType = UTypes.Undefined Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
@@ -267,25 +269,30 @@ Namespace API.Reddit
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download Base Functions"
|
||||
Private Sub DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False)
|
||||
Private Function DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False,
|
||||
Optional ByVal Title As String = Nothing) As Boolean
|
||||
Try
|
||||
Dim added As Boolean = False
|
||||
Dim cn$ = IIf(IsChannel, "media_metadata", "mediaMetadata")
|
||||
If Not w Is Nothing AndAlso w(cn).XmlIfNothing.Count > 0 Then
|
||||
Dim t As EContainer
|
||||
For Each n As EContainer In w(cn)
|
||||
t = n.ItemF({"s", "u"})
|
||||
If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel, Title), LNC)
|
||||
added = True
|
||||
If FirstOnly Then Exit For
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Return added
|
||||
Catch ex As Exception
|
||||
LogError(ex, "gallery parsing error")
|
||||
HasError = True
|
||||
Return False
|
||||
End Try
|
||||
End Sub
|
||||
End Function
|
||||
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
|
||||
Try
|
||||
ThrowAny(Token)
|
||||
@@ -333,13 +340,15 @@ Namespace API.Reddit
|
||||
#End Region
|
||||
#Region "Structure creator"
|
||||
Protected Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False) As UserMedia
|
||||
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False,
|
||||
Optional ByVal Title As String = Nothing) As UserMedia
|
||||
If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing
|
||||
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
|
||||
Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
|
||||
If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing
|
||||
If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
|
||||
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
|
||||
If Not Title.IsEmptyString Then m.Post.Title = Title
|
||||
Return m
|
||||
End Function
|
||||
Private Function TryFile(ByVal URL As String) As Boolean
|
||||
@@ -436,7 +445,11 @@ Namespace API.Reddit
|
||||
Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
|
||||
Case UTypes.Video, UTypes.m3u8 : DownloadedVideos += 1 : _CountVideo += 1
|
||||
End Select
|
||||
v.File = f
|
||||
If Not IsChannel Or Not SaveToCache Then
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
Else
|
||||
v.File = f
|
||||
End If
|
||||
v.Post.CachedFile = f
|
||||
v.State = UStates.Downloaded
|
||||
dCount += 1
|
||||
|
||||
@@ -47,7 +47,7 @@ Namespace API.Twitter
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
Dim PostID$ = String.Empty
|
||||
Dim PostDate$
|
||||
Dim PostDate$, PostTitle$
|
||||
Dim m As EContainer, nn As EContainer, s As EContainer
|
||||
Dim NewPostDetected As Boolean = False
|
||||
Dim ExistsDetected As Boolean = False
|
||||
@@ -83,17 +83,18 @@ Namespace API.Twitter
|
||||
ExistsDetected = True
|
||||
Continue For
|
||||
End If
|
||||
PostTitle = nn.Value("full_text")
|
||||
|
||||
If Not ParseUserMediaOnly OrElse (Not nn.Contains("retweeted_status") OrElse
|
||||
(Not ID.IsEmptyString AndAlso UID(nn("retweeted_status")) = ID)) Then
|
||||
If Not CheckVideoNode(nn, PostID, PostDate) Then
|
||||
If Not CheckVideoNode(nn, PostID, PostDate, PostTitle) Then
|
||||
s = nn.ItemF({"extended_entities", "media"})
|
||||
If s Is Nothing OrElse s.Count = 0 Then s = nn.ItemF({"retweeted_status", "extended_entities", "media"})
|
||||
If Not s Is Nothing AndAlso s.Count > 0 Then
|
||||
For Each m In s
|
||||
If m.Count > 0 AndAlso m.Contains("media_url") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(m("media_url").Value,
|
||||
PostID, PostDate, GetPictureOption(m)), LNC)
|
||||
PostID, PostDate, GetPictureOption(m), PostTitle), LNC)
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
@@ -158,10 +159,10 @@ Namespace API.Twitter
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Video options"
|
||||
Private Function CheckVideoNode(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String) As Boolean
|
||||
Private Function CheckVideoNode(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String, ByVal Title As String) As Boolean
|
||||
Try
|
||||
Dim URL$ = GetVideoNodeURL(w)
|
||||
If Not URL.IsEmptyString Then _TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate), LNC) : Return True
|
||||
If Not URL.IsEmptyString Then _TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate,, Title), LNC) : Return True
|
||||
Return False
|
||||
Catch ex As Exception
|
||||
LogError(ex, "[CheckVideoNode]")
|
||||
@@ -193,7 +194,7 @@ Namespace API.Twitter
|
||||
End Sub
|
||||
#End Region
|
||||
Private Shared Function MediaFromData(ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _PictureOption As String = "") As UserMedia
|
||||
Optional ByVal _PictureOption As String = "", Optional ByVal Title As String = Nothing) As UserMedia
|
||||
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
|
||||
Dim m As New UserMedia(_URL) With {.PictureOption = _PictureOption, .Post = New UserPost With {.ID = PostID}}
|
||||
If Not m.URL.IsEmptyString Then m.File = CStr(RegexReplace(m.URL, FilesPattern))
|
||||
@@ -201,6 +202,7 @@ Namespace API.Twitter
|
||||
m.URL_BASE = $"{m.URL.Replace($".{m.File.Extension}", String.Empty)}?format={m.File.Extension}&name={m.PictureOption}"
|
||||
End If
|
||||
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, Declarations.DateProvider, Nothing) Else m.Post.Date = Nothing
|
||||
If Not Title.IsEmptyString Then m.Post.Title = Title
|
||||
Return m
|
||||
End Function
|
||||
#End Region
|
||||
@@ -241,7 +243,7 @@ Namespace API.Twitter
|
||||
Case "mp4" : v.Type = UserMedia.Types.Video : DownloadedVideos += 1 : _CountVideo += 1
|
||||
Case Else : v.Type = UserMedia.Types.Picture : DownloadedPictures += 1 : _CountPictures += 1
|
||||
End Select
|
||||
v.File = f
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
v.State = UStates.Downloaded
|
||||
Catch wex As Exception
|
||||
ErrorDownloading(f, v.URL_BASE)
|
||||
|
||||
Reference in New Issue
Block a user