geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

1.0.0.3

Browse Source 
Fix typo in properties, re-enabling the "Download all" button after changing the mode, removed unused elements
This commit is contained in:
Andy
2021-12-11 07:44:43 +03:00
parent 4acef31c99
commit 70e73c7d97
8 changed files with 26 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Changelog.md
View File

@@ -1,3 +1,9 @@
# 1.0.0.3
- Fixed
- Custom "Download videos" option is not saved
- The "Download all" button is not activated after changing modes
# 1.0.0.2 # 1.0.0.2
- Added - Added

1
SCrawler/API/Base/Structures.vb
View File

@@ -49,7 +49,6 @@
''' <summary>Post ID</summary> ''' <summary>Post ID</summary>
Friend ID As String Friend ID As String
Friend [Date] As Date? Friend [Date] As Date?
Friend Title As String
#Region "Channel compatible fields" #Region "Channel compatible fields"
Friend UserID As String Friend UserID As String
Friend CachedFile As SFile Friend CachedFile As SFile

2
SCrawler/API/Base/UserDataBase.vb
View File

@@ -436,7 +436,7 @@ BlockNullPicture:
SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing) SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True) ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True) DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
DownloadedVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True) DownloadVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
_CountVideo = x.Value(Name_VideoCount).FromXML(Of Integer)(0) _CountVideo = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
_CountPictures = x.Value(Name_PicturesCount).FromXML(Of Integer)(0) _CountPictures = x.Value(Name_PicturesCount).FromXML(Of Integer)(0)
LastUpdated = AConvert(Of Date)(x.Value(Name_LastUpdated), ADateTime.Formats.BaseDateTime, Nothing) LastUpdated = AConvert(Of Date)(x.Value(Name_LastUpdated), ADateTime.Formats.BaseDateTime, Nothing)

22
SCrawler/API/Reddit/UserData.vb
View File

@@ -92,7 +92,7 @@ Namespace API.Reddit
Dim URL$ = String.Empty Dim URL$ = String.Empty
Try Try
Dim PostID$ = String.Empty Dim PostID$ = String.Empty
Dim PostDate$, PostTitle$ Dim PostDate$
Dim n As EContainer, nn As EContainer, s As EContainer Dim n As EContainer, nn As EContainer, s As EContainer
Dim NewPostDetected As Boolean = False Dim NewPostDetected As Boolean = False
Dim ExistsDetected As Boolean = False Dim ExistsDetected As Boolean = False
@@ -124,30 +124,29 @@ Namespace API.Reddit
ExistsDetected = True ExistsDetected = True
Continue For Continue For
End If End If
PostTitle = nn.Value("title")
If CheckNode(nn) Then If CheckNode(nn) Then
_ItemsBefore = _TempMediaList.Count _ItemsBefore = _TempMediaList.Count
added = True added = True
s = nn.ItemF({"source", "url"}) s = nn.ItemF({"source", "url"})
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC) _TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
Else Else
s = nn.ItemF({"media"}).XmlIfNothing s = nn.ItemF({"media"}).XmlIfNothing
__ItemType = s("type").XmlIfNothingValue __ItemType = s("type").XmlIfNothingValue
Select Case __ItemType Select Case __ItemType
Case "gallery" : If Not DownloadGallery(s, PostID, PostDate,,, PostTitle) Then added = False Case "gallery" : If Not DownloadGallery(s, PostID, PostDate) Then added = False
Case "image", "gifvideo" Case "image", "gifvideo"
If s.Contains("content") Then If s.Contains("content") Then
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"), _TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
PostID, PostDate,, IsChannel, PostTitle), LNC) PostID, PostDate,, IsChannel), LNC)
Else Else
added = False added = False
End If End If
Case "video" Case "video"
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"), _TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
PostID, PostDate,, IsChannel, PostTitle), LNC) PostID, PostDate,, IsChannel), LNC)
Else Else
added = False added = False
End If End If
@@ -167,7 +166,7 @@ Namespace API.Reddit
End Select End Select
End With End With
If Not tmpType = UTypes.Undefined Then If Not tmpType = UTypes.Undefined Then
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC) _TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
End If End If
End If End If
End If End If
@@ -270,8 +269,7 @@ Namespace API.Reddit
#End Region #End Region
#Region "Download Base Functions" #Region "Download Base Functions"
Private Function DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String, Private Function DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False, Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False) As Boolean
Optional ByVal Title As String = Nothing) As Boolean
Try Try
Dim added As Boolean = False Dim added As Boolean = False
Dim cn$ = IIf(IsChannel, "media_metadata", "mediaMetadata") Dim cn$ = IIf(IsChannel, "media_metadata", "mediaMetadata")
@@ -280,7 +278,7 @@ Namespace API.Reddit
For Each n As EContainer In w(cn) For Each n As EContainer In w(cn)
t = n.ItemF({"s", "u"}) t = n.ItemF({"s", "u"})
If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel, Title), LNC) _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel), LNC)
added = True added = True
If FirstOnly Then Exit For If FirstOnly Then Exit For
End If End If
@@ -340,15 +338,13 @@ Namespace API.Reddit
#End Region #End Region
#Region "Structure creator" #Region "Structure creator"
Protected Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String, Protected Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False, Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False) As UserMedia
Optional ByVal Title As String = Nothing) As UserMedia
If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern)) _URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}} Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing
If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}" If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
If Not Title.IsEmptyString Then m.Post.Title = Title
Return m Return m
End Function End Function
Private Function TryFile(ByVal URL As String) As Boolean Private Function TryFile(ByVal URL As String) As Boolean

14
SCrawler/API/Twitter/UserData.vb
View File

@@ -47,7 +47,7 @@ Namespace API.Twitter
Dim URL$ = String.Empty Dim URL$ = String.Empty
Try Try
Dim PostID$ = String.Empty Dim PostID$ = String.Empty
Dim PostDate$, PostTitle$ Dim PostDate$
Dim m As EContainer, nn As EContainer, s As EContainer Dim m As EContainer, nn As EContainer, s As EContainer
Dim NewPostDetected As Boolean = False Dim NewPostDetected As Boolean = False
Dim ExistsDetected As Boolean = False Dim ExistsDetected As Boolean = False
@@ -83,18 +83,17 @@ Namespace API.Twitter
ExistsDetected = True ExistsDetected = True
Continue For Continue For
End If End If
PostTitle = nn.Value("full_text")
If Not ParseUserMediaOnly OrElse (Not nn.Contains("retweeted_status") OrElse If Not ParseUserMediaOnly OrElse (Not nn.Contains("retweeted_status") OrElse
(Not ID.IsEmptyString AndAlso UID(nn("retweeted_status")) = ID)) Then (Not ID.IsEmptyString AndAlso UID(nn("retweeted_status")) = ID)) Then
If Not CheckVideoNode(nn, PostID, PostDate, PostTitle) Then If Not CheckVideoNode(nn, PostID, PostDate) Then
s = nn.ItemF({"extended_entities", "media"}) s = nn.ItemF({"extended_entities", "media"})
If s Is Nothing OrElse s.Count = 0 Then s = nn.ItemF({"retweeted_status", "extended_entities", "media"}) If s Is Nothing OrElse s.Count = 0 Then s = nn.ItemF({"retweeted_status", "extended_entities", "media"})
If Not s Is Nothing AndAlso s.Count > 0 Then If Not s Is Nothing AndAlso s.Count > 0 Then
For Each m In s For Each m In s
If m.Count > 0 AndAlso m.Contains("media_url") Then If m.Count > 0 AndAlso m.Contains("media_url") Then
_TempMediaList.ListAddValue(MediaFromData(m("media_url").Value, _TempMediaList.ListAddValue(MediaFromData(m("media_url").Value,
PostID, PostDate, GetPictureOption(m), PostTitle), LNC) PostID, PostDate, GetPictureOption(m)), LNC)
End If End If
Next Next
End If End If
@@ -159,10 +158,10 @@ Namespace API.Twitter
End Function End Function
#End Region #End Region
#Region "Video options" #Region "Video options"
Private Function CheckVideoNode(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String, ByVal Title As String) As Boolean Private Function CheckVideoNode(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String) As Boolean
Try Try
Dim URL$ = GetVideoNodeURL(w) Dim URL$ = GetVideoNodeURL(w)
If Not URL.IsEmptyString Then _TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate,, Title), LNC) : Return True If Not URL.IsEmptyString Then _TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate), LNC) : Return True
Return False Return False
Catch ex As Exception Catch ex As Exception
LogError(ex, "[CheckVideoNode]") LogError(ex, "[CheckVideoNode]")
@@ -194,7 +193,7 @@ Namespace API.Twitter
End Sub End Sub
#End Region #End Region
Private Shared Function MediaFromData(ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String, Private Shared Function MediaFromData(ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
Optional ByVal _PictureOption As String = "", Optional ByVal Title As String = Nothing) As UserMedia Optional ByVal _PictureOption As String = "") As UserMedia
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern)) _URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
Dim m As New UserMedia(_URL) With {.PictureOption = _PictureOption, .Post = New UserPost With {.ID = PostID}} Dim m As New UserMedia(_URL) With {.PictureOption = _PictureOption, .Post = New UserPost With {.ID = PostID}}
If Not m.URL.IsEmptyString Then m.File = CStr(RegexReplace(m.URL, FilesPattern)) If Not m.URL.IsEmptyString Then m.File = CStr(RegexReplace(m.URL, FilesPattern))
@@ -202,7 +201,6 @@ Namespace API.Twitter
m.URL_BASE = $"{m.URL.Replace($".{m.File.Extension}", String.Empty)}?format={m.File.Extension}&name={m.PictureOption}" m.URL_BASE = $"{m.URL.Replace($".{m.File.Extension}", String.Empty)}?format={m.File.Extension}&name={m.PictureOption}"
End If End If
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, Declarations.DateProvider, Nothing) Else m.Post.Date = Nothing If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, Declarations.DateProvider, Nothing) Else m.Post.Date = Nothing
If Not Title.IsEmptyString Then m.Post.Title = Title
Return m Return m
End Function End Function
#End Region #End Region

2
SCrawler/Editors/UserCreatorForm.vb
View File

@@ -106,7 +106,7 @@ Namespace Editors
CH_PARSE_USER_MEDIA.Checked = .ParseUserMediaOnly CH_PARSE_USER_MEDIA.Checked = .ParseUserMediaOnly
CH_READY_FOR_DOWN.Checked = .ReadyForDownload CH_READY_FOR_DOWN.Checked = .ReadyForDownload
CH_DOWN_IMAGES.Checked = .DownloadImages CH_DOWN_IMAGES.Checked = .DownloadImages
CH_DOWN_VIDEOS.Checked = .DownloadedVideos CH_DOWN_VIDEOS.Checked = .DownloadVideos
TXT_DESCR.Text = .Description TXT_DESCR.Text = .Description
UserLabels.ListAddList(.Labels) UserLabels.ListAddList(.Labels)
If UserLabels.ListExists Then TXT_LABELS.Text = UserLabels.ListToString If UserLabels.ListExists Then TXT_LABELS.Text = UserLabels.ListToString

2
SCrawler/MainFrame.vb
View File

@@ -405,7 +405,7 @@ CloseResume:
End If End If
End If End If
Settings.ShowingMode.Value = m Settings.ShowingMode.Value = m
If Not m = ShowingModes.All Then BTT_DOWN_ALL.Enabled = False BTT_DOWN_ALL.Enabled = m = ShowingModes.All
End Sub End Sub
Private Sub BTT_SELECT_LABELS_Click(sender As Object, e As EventArgs) Handles BTT_SELECT_LABELS.Click Private Sub BTT_SELECT_LABELS_Click(sender As Object, e As EventArgs) Handles BTT_SELECT_LABELS.Click
OpenLabelsForm() OpenLabelsForm()

4
SCrawler/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below: ' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")> ' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("1.0.0.2")> <Assembly: AssemblyVersion("1.0.0.3")>
<Assembly: AssemblyFileVersion("1.0.0.2")> <Assembly: AssemblyFileVersion("1.0.0.3")>
<Assembly: NeutralResourcesLanguage("en")> <Assembly: NeutralResourcesLanguage("en")>