mid-kid
/
RocketLinux
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Say hello to the all-new freshnaut. Now with much better accuracy.

master
mid-kid 9 years ago
parent
11021ce341
commit
38a9c042af
3 changed files with 350 additions and 160 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 337
      astronaut/freshnaut
  2. 161
      graveyard/freshnaut
  3. 6
      graveyard/freshnaut.cause_of_death

337
astronaut/freshnaut
View File

@ -1,161 +1,184 @@
#!/bin/sh -e
# A script to check for satellite updates.
# It just downloads the documents which should be updated when the software has an update.
# It's a shitty method, but it kinda works... Other than having a lot of false-positives.
_show_help() {
echo "This program will help keep your satellites fresh and up to date.
Usage: $0 <[\$dir_satellites] [\$_dir_fresh]|ignore <diff> <ignorediff>>
\$dir_satellites = The directory where all the satellites are you want to update.
\$_dir_fresh = The directory that will be used to keep all the files for checking.
ignore = Use this option to generate an ignorediff.
diff = A diff file. Generated without -u or any other fancy options.
ignorediff = A file containing the differences to be ignored (this file will be appended to).
The variables (starting with \$) can also be set from a configuration file."
}
get_variable() {
printf "
download() { :; }
extract() { :; }
getfile() { :; }
extrafile() { :; }
dlextract() { :; }
dlfile() { :; }
header_end() { echo \$$1; exit; }
. %s
" $2 | sh
}
download() {
$(echo "$cmd_download" | sed -e 's@{dst}@'"$2"'@g' -e 's@{src}@'"$1"'@g') 2> /dev/null
}
diff_extractlines() {
# Extract all the line numbers from a diff.
stage=0
for line in $(echo "$1" | awk '{print $1}'); do
if [ "$stage" = 0 ]; then
if [ "$line" != ">" -a "$line" != "<" ]; then
echo "$line"
stage=1
fi
elif [ "$stage" = 1 ]; then
if [ "$line" != ">" -a "$line" != "<" ]; then
if [ "$line" = "---" ]; then
stage=0
#!/usr/bin/env lua
function show_help()
print(string.format([[This program will help you keep your satellites fresh and up to date.
Usage: %s <sat|dir> <check> <save>
sat = Check only one satellite
dir = Check a dir full of satellites
check = What file (sat) or directory (dir) to check
save = What file (sat) or directory (dir) to save known versions into]], arg[0]))
end
function call(command)
--[[--
-- Runs a command, returns all output except trailing newline on success.
--]]--
local pipe = io.popen(command)
if pipe then
local output = pipe:read("a")
if pipe:close() then
if #output > 0 then
return output:match("(.-)%s*$")
end
end
end
end
function astrohelp(satellite, func)
-- satellite_dir is an example of me being lazy and wanting to keep output clean in other functions.
if satellite_dir then
return call("astrohelp '" .. satellite_dir .. "/" .. satellite .. "' " .. func)
else
echo "$line"
fi
fi
fi
done
}
apply_ignorediff() {
# Filter a diff, only return the differences that shouldn't be ignored.
# TODO: This function is slow, due to the usage of awk on every line. A solution for this problem would be great.
stage=0
ignore=0
echo "$1" | while read line; do
first="$(echo "$line" | awk '{print $1}')"
if [ "$stage" = 0 ]; then
if [ "$first" != ">" -a "$first" != "<" ]; then
stage=1
ignore=0
if grep -e "^$line\$" "$2" > /dev/null; then
ignore=1
fi
fi
elif [ "$stage" = 1 ]; then
if [ "$first" != ">" -a "$first" != "<" ]; then
if [ "$line" = "---" ]; then
stage=0
return call("astrohelp '" .. satellite .. "' " .. func)
end
end
function download(url)
return call("curl -Ls '" .. url .. "'")
end
function make_regex(filename, version)
--[[--
-- Converts a string into regex, by escaping all special (magic) characters.
-- Returns two regexes, meant to be used in a loop. (See code using this function to know what I mean)
--]]--
local regex = filename:gsub(version, "{{version}}"):gsub("%%", "%%%%")
-- http://www.lua.org/manual/5.3/manual.html#6.4.1
local magic = "^$().[]*+-?"
for i = 1, #magic do
local char = magic:sub(i, i)
regex = regex:gsub("%" .. char, "%%" .. char)
end
return regex:gsub("{{version}}", "(.-") .. ")", regex:gsub(".*{{version}}", "(.-)")
end
function get_available_versions(satellite)
--[[--
-- Returns a set table containing all available versions on update_url
--]]--
local version = astrohelp(satellite, "variable version")
local update_url = astrohelp(satellite, "variable update_url")
if not version or not update_url then
return
end
local update_file = download(update_url)
if not update_file then
print("WARNING: " .. satellite .. ": Failed to download " .. update_url)
return
end
-- Try getting variable update_names, get downloads otherwise.
local update_names = astrohelp(satellite, "variable update_names")
if not update_names then
local tmp_update_names = astrohelp(satellite, "downloads")
assert(tmp_update_names, "Could not get update names for " .. satellite)
update_names = ""
-- Take the basename of every download url
for name in tmp_update_names:gmatch("[^\n]+") do
update_names = update_names .. name:match("[^/]+$") .. ","
end
end
local versions = {}
local count = 0
for update_name in update_names:gmatch("[^,]+") do
local loop_regex, end_regex = make_regex(update_name, version)
for x in update_file:gmatch(loop_regex) do
-- Sometimes, the match starts way too soon, and we get a very long "version" number.
-- Here we make sure we really get the shortest possible match.
local version
repeat
version = x
x = x:match(loop_regex)
until not x
version = version:match(end_regex)
versions[version] = true
count = count + 1
end
end
if count < 1 then
print("WARNING: " .. satellite .. ": Couldn't retrieve any versions")
return
elseif not versions[version] then
print("WARNING: " .. satellite .. ": Current version isn't available")
end
return versions
end
function check_new_versions(satellite, version_file_path)
--[[--
-- Check if new versions are available for a satellite.
-- Logs new versions to log_file, which should be a file descriptor.
-- Uses version_file_path to know where to store all known versions.
--]]--
local versions = get_available_versions(satellite)
if not versions then
return
end
local version_file = io.open(version_file_path, "r+")
local old_versions = nil
if version_file then
old_versions = {}
for version in version_file:lines() do
old_versions[version] = true
end
else
ignore=0
if grep -e "^$line\$" "$2" > /dev/null; then
ingore=1
fi
fi
fi
fi
if [ "$ignore" = 0 ]; then
echo "$line"
fi
done
}
# Do something completely different when this option is provided.
if [ "$1" = "ignore" ]; then
if [ $# -lt 3 ]; then
_show_help
exit 1
fi
diff_extractlines "$(cat "$2")" >> "$3"
# Don't do anything else
exit
fi
# Load the config
cmd_download="curl -L -o {dst} {src}" # Command to execute to download files
[ -f /etc/astronaut.conf ] && . /etc/astronaut.conf
[ -f "$HOME/.astronaut.conf" ] && . "$HOME/.astronaut.conf"
# Override config with command-line parameters.
[ "$1" ] && dir_satellites="$1"
[ "$2" ] && _dir_fresh="$2"
if [ ! "$dir_satellites" -o ! "$_dir_fresh" ]; then
_show_help
exit 1
fi
mkdir -p "$_dir_fresh"
for sat in "$dir_satellites"/*.sat; do
update_url="$(get_variable update_url "$sat")"
name="$(basename "$sat" .sat)"
if [ "$update_url" ]; then
printf "Checking: $name..."
cols="$(expr $(tput cols) - $(printf "Checking: $name..." | wc -c))"
download "$update_url" "$_dir_fresh/$name.fresh"
if [ -f "$_dir_fresh/$name" ]; then
check="$(diff "$_dir_fresh/$name" "$_dir_fresh/$name.fresh" || true)"
if [ -f "$_dir_fresh/$name.ignorediff" ]; then
check="$(apply_ignorediff "$check" "$_dir_fresh/$name.ignorediff")"
fi
if [ "$check" ]; then
printf "%${cols}s\n" "Update detected"
echo "$check" >> "$_dir_fresh/$name-$(date +%Y%m%d%H%M%S).diff"
print(satellite .. ": First check")
version_file = io.open(version_file_path, "w")
assert(version_file, "Failed to create " .. version_file_path)
end
for version, _ in pairs(versions) do
if old_versions then
if not old_versions[version] then
print(satellite .. ": " .. version)
version_file:write(version .. "\n")
end
else
printf "\n"
fi
print(satellite .. ": " .. version)
version_file:write(version .. "\n")
end
end
version_file:close()
end
if #arg < 3 then
show_help()
os.exit(1)
end
if arg[1] == "sat" then
check_new_versions(arg[2], arg[3])
elseif arg[1] == "dir" then
satellite_dir = arg[2]
local version_dir = arg[3]
assert(os.execute("mkdir -p " .. version_dir), "Failed to create " .. version_dir)
local find = io.popen("find " .. satellite_dir .. " -type f -name '*.sat' -printf '%P\n'")
assert(find, "Failed to run find")
for satellite in find:lines() do
local version_file = version_dir .. "/" .. satellite
if os.execute("mkdir -p " .. version_file:match("^.+/")) then
check_new_versions(satellite, version_file)
else
printf "%${cols}s\n" "First update check, creating new file"
download "$update_url" "$_dir_fresh/$name.checkfresh"
check="$(diff "$_dir_fresh/$name.fresh" "$_dir_fresh/$name.checkfresh" || true)"
if [ "$check" ]; then
echo "$(diff_extractlines "$check")" >> "$_dir_fresh/$name.ignorediff"
echo "> Created ignorediff"
fi
rm "$_dir_fresh/$name.checkfresh"
fi
mv "$_dir_fresh/$name.fresh" "$_dir_fresh/$name"
fi
done
print("Failed to create dir for " .. satellite)
end
end
end

161
graveyard/freshnaut
View File

@ -0,0 +1,161 @@
#!/bin/sh -e
# A script to check for satellite updates.
# It just downloads the documents which should be updated when the software has an update.
# It's a shitty method, but it kinda works... Other than having a lot of false-positives.
_show_help() {
echo "This program will help keep your satellites fresh and up to date.
Usage: $0 <[\$dir_satellites] [\$_dir_fresh]|ignore <diff> <ignorediff>>
\$dir_satellites = The directory where all the satellites are you want to update.
\$_dir_fresh = The directory that will be used to keep all the files for checking.
ignore = Use this option to generate an ignorediff.
diff = A diff file. Generated without -u or any other fancy options.
ignorediff = A file containing the differences to be ignored (this file will be appended to).
The variables (starting with \$) can also be set from a configuration file."
}
get_variable() {
printf "
download() { :; }
extract() { :; }
getfile() { :; }
extrafile() { :; }
dlextract() { :; }
dlfile() { :; }
header_end() { echo \$$1; exit; }
. %s
" $2 | sh
}
download() {
$(echo "$cmd_download" | sed -e 's@{dst}@'"$2"'@g' -e 's@{src}@'"$1"'@g') 2> /dev/null
}
diff_extractlines() {
# Extract all the line numbers from a diff.
stage=0
for line in $(echo "$1" | awk '{print $1}'); do
if [ "$stage" = 0 ]; then
if [ "$line" != ">" -a "$line" != "<" ]; then
echo "$line"
stage=1
fi
elif [ "$stage" = 1 ]; then
if [ "$line" != ">" -a "$line" != "<" ]; then
if [ "$line" = "---" ]; then
stage=0
else
echo "$line"
fi
fi
fi
done
}
apply_ignorediff() {
# Filter a diff, only return the differences that shouldn't be ignored.
# TODO: This function is slow, due to the usage of awk on every line. A solution for this problem would be great.
stage=0
ignore=0
echo "$1" | while read line; do
first="$(echo "$line" | awk '{print $1}')"
if [ "$stage" = 0 ]; then
if [ "$first" != ">" -a "$first" != "<" ]; then
stage=1
ignore=0
if grep -e "^$line\$" "$2" > /dev/null; then
ignore=1
fi
fi
elif [ "$stage" = 1 ]; then
if [ "$first" != ">" -a "$first" != "<" ]; then
if [ "$line" = "---" ]; then
stage=0
else
ignore=0
if grep -e "^$line\$" "$2" > /dev/null; then
ingore=1
fi
fi
fi
fi
if [ "$ignore" = 0 ]; then
echo "$line"
fi
done
}
# Do something completely different when this option is provided.
if [ "$1" = "ignore" ]; then
if [ $# -lt 3 ]; then
_show_help
exit 1
fi
diff_extractlines "$(cat "$2")" >> "$3"
# Don't do anything else
exit
fi
# Load the config
cmd_download="curl -L -o {dst} {src}" # Command to execute to download files
[ -f /etc/astronaut.conf ] && . /etc/astronaut.conf
[ -f "$HOME/.astronaut.conf" ] && . "$HOME/.astronaut.conf"
# Override config with command-line parameters.
[ "$1" ] && dir_satellites="$1"
[ "$2" ] && _dir_fresh="$2"
if [ ! "$dir_satellites" -o ! "$_dir_fresh" ]; then
_show_help
exit 1
fi
mkdir -p "$_dir_fresh"
for sat in "$dir_satellites"/*.sat; do
update_url="$(get_variable update_url "$sat")"
name="$(basename "$sat" .sat)"
if [ "$update_url" ]; then
printf "Checking: $name..."
cols="$(expr $(tput cols) - $(printf "Checking: $name..." | wc -c))"
download "$update_url" "$_dir_fresh/$name.fresh"
if [ -f "$_dir_fresh/$name" ]; then
check="$(diff "$_dir_fresh/$name" "$_dir_fresh/$name.fresh" || true)"
if [ -f "$_dir_fresh/$name.ignorediff" ]; then
check="$(apply_ignorediff "$check" "$_dir_fresh/$name.ignorediff")"
fi
if [ "$check" ]; then
printf "%${cols}s\n" "Update detected"
echo "$check" >> "$_dir_fresh/$name-$(date +%Y%m%d%H%M%S).diff"
else
printf "\n"
fi
else
printf "%${cols}s\n" "First update check, creating new file"
download "$update_url" "$_dir_fresh/$name.checkfresh"
check="$(diff "$_dir_fresh/$name.fresh" "$_dir_fresh/$name.checkfresh" || true)"
if [ "$check" ]; then
echo "$(diff_extractlines "$check")" >> "$_dir_fresh/$name.ignorediff"
echo "> Created ignorediff"
fi
rm "$_dir_fresh/$name.checkfresh"
fi
mv "$_dir_fresh/$name.fresh" "$_dir_fresh/$name"
fi
done

6
graveyard/freshnaut.cause_of_death
View File

@ -0,0 +1,6 @@
Freshnaut died on 19 december 2015.
It's latest known update was 4 november of the same year.
Birth date is unknown.
Cause of death:
I've started working on freshnaut as a way of checking software for updates with as little missed updates as possible. At the time, I thought regex methods were prone to missing a lot of updates, and while they do, this method is even worse. Yes, diffing websites is hell. The only websites this worked on were things like palemoon, but even that one updated a few times for halloween and other unnecessary things. As soon as you get into generated websites like sourceforge, everything changes every time you fetch it, and even if you try to ignore some ever-changing lines, you'll start noticing every single update they do to their site anyway (they even have some line in the file which tells the page's version). So, I've started doing a different approach (this time in lua), which relies on regex shenanigans, but that's all handled by the implementation, as the satellite writer doesn't have to worry about it too much.
Write Preview
Loading…
Cancel
Save