geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2022.9.13.0

Browse Source 
Added video duration to the feed
Added skipping of pinned Instagram posts if they are already downloaded
This commit is contained in:
Andy
2022-09-13 16:20:07 +03:00
parent c28c0e1ba3
commit 9567b0a367
10 changed files with 66 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Changelog.md
View File

@@ -1,3 +1,13 @@
# 2022.9.13.0
*2022-09-13*
- Added
- Video duration to the feed
- Fixed
- (Issue #70) Instagram posts not downloading if there are pinned posts that have already been downloaded
- Minor bugs
# 2022.9.10.0
*2022-09-10*

7
SCrawler/API/Base/UserDataBase.vb
View File

@@ -830,7 +830,7 @@ BlockNullPicture:
Else
'ReparseMissing(Token)
End If
_TempMediaList.ListAddList(ContentMissing, LNC)
'_TempMediaList.ListAddList(ContentMissing, LNC)
If _TempMediaList.Count > 0 Then
If Not DownloadImages Then _TempMediaList.RemoveAll(Function(m) m.Type = UTypes.GIF Or m.Type = UTypes.Picture)
@@ -845,9 +845,12 @@ BlockNullPicture:
_ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
DownloadContent(Token)
ThrowIfDisposed()
LatestData.ListAddList(_ContentNew.Where(_downContent), LNC)
Dim mcb& = If(ContentMissingExists, _ContentList.LongCount(Function(c) MissingFinder(c)), 0)
_ContentList.ListAddList(_ContentNew.Where(Function(c) _downContent(c) Or MissingFinder(c)), LNC)
If DownloadedTotal(False) > 0 Or EnvirChanged.Invoke Or _ContentList.Exists(MissingFinder) Then
Dim mca& = If(ContentMissingExists, _ContentList.LongCount(Function(c) MissingFinder(c)), 0)
If DownloadedTotal(False) > 0 Or EnvirChanged.Invoke Or Not mcb = mca Then
If __SaveData Then
LastUpdated = Now
RunScript()

8
SCrawler/API/Instagram/UserData.vb
View File

@@ -247,6 +247,7 @@ Namespace API.Instagram
Try
Dim n As EContainer, nn As EContainer, node As EContainer
Dim HasNextPage As Boolean = False
Dim Pinned As Boolean
Dim EndCursor$ = String.Empty
Dim PostID$ = String.Empty, PostDate$ = String.Empty, SpecFolder$ = String.Empty
Dim TaggedCount%
@@ -296,7 +297,7 @@ Namespace API.Instagram
RequestsCount += 1
ThrowAny(Token)
'Data
'Parsing
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing
n = j.ItemF(ENode).XmlIfNothing
@@ -321,13 +322,14 @@ Namespace API.Instagram
End If
End If
PostID = node.Value("id")
If Not PostID.IsEmptyString And _TempPostsList.Contains(PostID) Then Throw New ExitException(_DownloadComplete)
Pinned = CBool(If(node("pinned_for_users")?.Count, 0))
If Not PostID.IsEmptyString And _TempPostsList.Contains(PostID) And Not Pinned Then Throw New ExitException(_DownloadComplete)
_TempPostsList.Add(PostID)
PostDate = node.Value("taken_at_timestamp")
If IsSavedPosts Then
_SavedPostsIDs.Add(PostID)
Else
If Not CheckDatesLimit(PostDate, DateProvider) Then Throw New ExitException(_DownloadComplete)
If Not CheckDatesLimit(PostDate, DateProvider) And Not Pinned Then Throw New ExitException(_DownloadComplete)
ObtainMedia(node, PostID, PostDate, SpecFolder)
End If
Next

1
SCrawler/Download/FeedMedia.vb
View File

@@ -131,6 +131,7 @@ Namespace DownloadObjects
End With
End If
If Not MyVideo Is Nothing Then info &= $" ({MyVideo.VideoLength})"
LBL_INFO.Text = info
'TT_MAIN.SetToolTip(LBL_INFO, Information)
s = New Size(Width, h + TP_MAIN.RowStyles(0).Height + PaddingE.GetOf({TP_MAIN}).Vertical(2))

13
SCrawler/Download/FeedVideo.Designer.vb generated
View File

@@ -30,6 +30,7 @@ Namespace DownloadObjects
Me.BTT_PAUSE = New System.Windows.Forms.Button()
Me.BTT_STOP = New System.Windows.Forms.Button()
Me.TR_VOLUME = New System.Windows.Forms.TrackBar()
Me.LBL_TIME = New System.Windows.Forms.Label()
TP_MAIN = New System.Windows.Forms.TableLayoutPanel()
TP_BUTTONS = New System.Windows.Forms.TableLayoutPanel()
TP_MAIN.SuspendLayout()
@@ -88,6 +89,7 @@ Namespace DownloadObjects
TP_BUTTONS.Controls.Add(Me.BTT_PAUSE, 1, 0)
TP_BUTTONS.Controls.Add(Me.BTT_STOP, 2, 0)
TP_BUTTONS.Controls.Add(Me.TR_VOLUME, 4, 0)
TP_BUTTONS.Controls.Add(Me.LBL_TIME, 3, 0)
TP_BUTTONS.Dock = System.Windows.Forms.DockStyle.Fill
TP_BUTTONS.Location = New System.Drawing.Point(1, 133)
TP_BUTTONS.Margin = New System.Windows.Forms.Padding(1)
@@ -141,6 +143,16 @@ Namespace DownloadObjects
Me.TR_VOLUME.Size = New System.Drawing.Size(94, 20)
Me.TR_VOLUME.TabIndex = 3
'
'LBL_TIME
'
Me.LBL_TIME.AutoSize = True
Me.LBL_TIME.Dock = System.Windows.Forms.DockStyle.Fill
Me.LBL_TIME.Location = New System.Drawing.Point(78, 0)
Me.LBL_TIME.Name = "LBL_TIME"
Me.LBL_TIME.Size = New System.Drawing.Size(1, 26)
Me.LBL_TIME.TabIndex = 4
Me.LBL_TIME.TextAlign = System.Drawing.ContentAlignment.MiddleCenter
'
'FeedVideo
'
Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
@@ -164,5 +176,6 @@ Namespace DownloadObjects
Private WithEvents BTT_PAUSE As Button
Private WithEvents BTT_STOP As Button
Private WithEvents TR_VOLUME As TrackBar
Private WithEvents LBL_TIME As Label
End Class
End Namespace

9
SCrawler/Download/FeedVideo.vb
View File

@@ -18,7 +18,14 @@ Namespace DownloadObjects
Dim v# = DivideWithZeroChecking(MediaPlayer.Time, MediaPlayer.Length) * 10
If v > 10 Then TR_POSITION.Value = 10 Else TR_POSITION.Value = v
End Sub
Private ReadOnly TimeChangeLabel As Action = Sub()
If MediaPlayer.Time >= 0 Then
Dim t As TimeSpan = TimeSpan.FromMilliseconds(MediaPlayer.Time)
LBL_TIME.Text = $"{VideoLength}/{t}"
End If
End Sub
Private ReadOnly MyImage As ImageRenderer
Friend ReadOnly VideoLength As TimeSpan
Public Sub New()
InitializeComponent()
End Sub
@@ -31,6 +38,7 @@ Namespace DownloadObjects
MediaPlayer = New [Shared].MediaPlayer(New [Shared].Media(New [Shared].LibVLC(enableDebugLogs:=debugLogs), New Uri(File.ToString)))
MyVideo.MediaPlayer = MediaPlayer
TR_VOLUME.Value = MediaPlayer.Volume / 10
If MediaPlayer.Length >= 0 Then VideoLength = TimeSpan.FromMilliseconds(MediaPlayer.Length)
If Settings.UseM3U8 Then
Dim f As SFile = $"{Settings.CachePath.PathWithSeparator}FeedSnapshots\{File.GetHashCode}.png"
If Not f.Exists Then f = FFMPEG.TakeSnapshot(File, f, Settings.FfmpegFile, TimeSpan.FromSeconds(1))
@@ -58,6 +66,7 @@ Namespace DownloadObjects
End Sub
Private Sub MediaPlayer_TimeChanged(sender As Object, e As [Shared].MediaPlayerTimeChangedEventArgs) Handles MediaPlayer.TimeChanged
If TR_POSITION.InvokeRequired Then TR_POSITION.Invoke(TimeChange) Else TimeChange.Invoke
If LBL_TIME.InvokeRequired Then LBL_TIME.Invoke(TimeChangeLabel) Else TimeChangeLabel.Invoke
End Sub
Private Sub TR_POSITION_MouseUp(sender As Object, e As MouseEventArgs) Handles TR_POSITION.MouseUp
Try : MediaPlayer.Time = (MediaPlayer.Length / 100) * (TR_POSITION.Value * 10) : Catch : End Try

25
SCrawler/MainFrame.Designer.vb generated
View File

@@ -40,6 +40,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
Me.BTT_DELETE_USER = New System.Windows.Forms.ToolStripButton()
Me.BTT_REFRESH = New System.Windows.Forms.ToolStripButton()
Me.BTT_SHOW_INFO = New System.Windows.Forms.ToolStripButton()
Me.BTT_FEED = New System.Windows.Forms.ToolStripButton()
Me.BTT_CHANNELS = New System.Windows.Forms.ToolStripButton()
Me.BTT_DOWN_SAVED = New System.Windows.Forms.ToolStripButton()
Me.BTT_DOWN_SELECTED = New System.Windows.Forms.ToolStripButton()
@@ -105,7 +106,6 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
Me.BTT_TRAY_SHOW_HIDE = New System.Windows.Forms.ToolStripMenuItem()
Me.BTT_TRAY_CLOSE = New System.Windows.Forms.ToolStripMenuItem()
Me.BTT_TRAY_CLOSE_NO_SCRIPT = New System.Windows.Forms.ToolStripMenuItem()
Me.BTT_FEED = New System.Windows.Forms.ToolStripButton()
SEP_1 = New System.Windows.Forms.ToolStripSeparator()
SEP_2 = New System.Windows.Forms.ToolStripSeparator()
CONTEXT_SEP_1 = New System.Windows.Forms.ToolStripSeparator()
@@ -147,7 +147,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
'MENU_SETTINGS_SEP_1
'
MENU_SETTINGS_SEP_1.Name = "MENU_SETTINGS_SEP_1"
MENU_SETTINGS_SEP_1.Size = New System.Drawing.Size(113, 6)
MENU_SETTINGS_SEP_1.Size = New System.Drawing.Size(177, 6)
'
'SEP_3
'
@@ -228,7 +228,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
'
Me.BTT_SETTINGS.Image = Global.SCrawler.My.Resources.Resources.SettingsPic_16
Me.BTT_SETTINGS.Name = "BTT_SETTINGS"
Me.BTT_SETTINGS.Size = New System.Drawing.Size(116, 22)
Me.BTT_SETTINGS.Size = New System.Drawing.Size(180, 22)
Me.BTT_SETTINGS.Text = "Settings"
'
'Toolbar_TOP
@@ -269,12 +269,12 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
'
'BTT_REFRESH
'
Me.BTT_REFRESH.AutoToolTip = False
Me.BTT_REFRESH.Image = Global.SCrawler.My.Resources.Resources.Refresh
Me.BTT_REFRESH.ImageTransparentColor = System.Drawing.Color.Magenta
Me.BTT_REFRESH.Name = "BTT_REFRESH"
Me.BTT_REFRESH.Size = New System.Drawing.Size(66, 22)
Me.BTT_REFRESH.Text = "Refresh"
Me.BTT_REFRESH.ToolTipText = "Refresh user list"
'
'BTT_SHOW_INFO
'
@@ -285,6 +285,15 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
Me.BTT_SHOW_INFO.Text = "Info"
Me.BTT_SHOW_INFO.ToolTipText = "Left-click: open the 'Info' form (show download summary)." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "Right click: open the " &
"'Missing' form (show information about missing posts)."
'
'BTT_FEED
'
Me.BTT_FEED.Image = Global.SCrawler.My.Resources.Resources.RSSPic
Me.BTT_FEED.ImageTransparentColor = System.Drawing.Color.Magenta
Me.BTT_FEED.Name = "BTT_FEED"
Me.BTT_FEED.Size = New System.Drawing.Size(52, 22)
Me.BTT_FEED.Text = "Feed"
Me.BTT_FEED.ToolTipText = "Feed of recently downloaded data"
'
'BTT_CHANNELS
'
@@ -776,14 +785,6 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
Me.BTT_TRAY_CLOSE_NO_SCRIPT.ToolTipText = "Close the program without executing the script"
Me.BTT_TRAY_CLOSE_NO_SCRIPT.Visible = False
'
'BTT_FEED
'
Me.BTT_FEED.Image = Global.SCrawler.My.Resources.Resources.RSSPic
Me.BTT_FEED.ImageTransparentColor = System.Drawing.Color.Magenta
Me.BTT_FEED.Name = "BTT_FEED"
Me.BTT_FEED.Size = New System.Drawing.Size(52, 22)
Me.BTT_FEED.Text = "Feed"
'
'MainFrame
'
Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)

8
SCrawler/MainFrame.vb
View File

@@ -1171,10 +1171,10 @@ ResumeDownloadingOperation:
Dim a As Action = Sub()
Dim i% = LIST_PROFILES.Items.IndexOfKey(Key)
If i < 0 Then
i = Settings.Users.FindIndex(Function(u) u.Key = Key)
If i >= 0 Then
UserListUpdate(Settings.Users(i), True)
i = LIST_PROFILES.Items.IndexOfKey(Key)
Dim u As IUserData = Settings.GetUser(Key, True)
If Not u Is Nothing Then
UserListUpdate(u, True)
i = LIST_PROFILES.Items.IndexOfKey(u.Key)
End If
End If
If i >= 0 Then

4
SCrawler/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2022.9.10.0")>
<Assembly: AssemblyFileVersion("2022.9.10.0")>
<Assembly: AssemblyVersion("2022.9.13.0")>
<Assembly: AssemblyFileVersion("2022.9.13.0")>
<Assembly: NeutralResourcesLanguage("en")>

8
SCrawler/SettingsCLS.vb
View File

@@ -343,7 +343,7 @@ Friend Class SettingsCLS : Implements IDisposable
_UserListUpdateRequired = True
End Try
End Sub
Friend Overloads Function GetUser(ByVal User As IUserData) As IUserData
Friend Overloads Function GetUser(ByVal User As IUserData, Optional ByVal GetCollection As Boolean = False) As IUserData
If Users.Count > 0 Then
Dim uSimple As Predicate(Of IUserData) = Function(u) u.Equals(DirectCast(User, UserDataBase))
Dim uCol As Predicate(Of IUserData) = Function(ByVal u As IUserData) As Boolean
@@ -360,7 +360,7 @@ Friend Class SettingsCLS : Implements IDisposable
If Users(i).IsCollection Then
With DirectCast(Users(i), UserDataBind)
i = .Collections.FindIndex(uSimple)
If i >= 0 Then Return .Collections(i)
If i >= 0 Then Return If(GetCollection, Users(i), .Collections(i))
End With
Else
Return Users(i)
@@ -369,7 +369,7 @@ Friend Class SettingsCLS : Implements IDisposable
End If
Return Nothing
End Function
Friend Overloads Function GetUser(ByVal UserKey As String) As IUserData
Friend Overloads Function GetUser(ByVal UserKey As String, Optional ByVal GetCollection As Boolean = False) As IUserData
If Users.Count > 0 Then
Dim finder As Predicate(Of IUserData) = Function(u) u.Key = UserKey
Dim i%, ii%
@@ -379,7 +379,7 @@ Friend Class SettingsCLS : Implements IDisposable
With DirectCast(.Self, UserDataBind)
If .Count > 0 Then
ii = .Collections.FindIndex(finder)
If ii >= 0 Then Return .Collections(ii)
If ii >= 0 Then Return If(GetCollection, .Self, .Collections(ii))
End If
End With
Else