beenull/bashblog
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More code refactoring. Limit word splitting and disable globbing by default.

Browse source 
- Globally, now do word splitting (IFS) only on newline (which also makes
  "$*" expand with newline separator instead of space).

- Disable globbing (pathmame expansion), to be re-enabled locally using
  'set +f' where needed (typically in a subshell).

These changes help eliminate unexpected snags and security vulnerabilities
in case someone forgets to quote a variable somewhere. They should also make
the code "just work" with spaces and other special characters in file names
and tags (as long as they're not newline characters, but that can't happen
with regular use of the script as the newline is the separator). This means
that, as of this change, editing or completely emptying the convert_filename
filter should no longer pose any problems as far as bb.sh is concerned.

The changes to adapt the code to the above are mainly:

- Now that we do word splitting on newline only, we can go back to iterating
  through files in a "for" loop instead of using "read" with a here-document,
  which is more readable. However, to enable globbing locally, a technique
  adaptation is needed, like:
     for file in $(set +f; printf '%s\n' *.html)
  or
     for file in $(set +f; ls -t -- *.html)  # sort by date, newest first
  Given IFS=$'\n' and globbing disabled globally, this technique is robust
  for all special characters in file names except for newlines.

- invoke_editor() function replaces direct $EDITOR calls, because we need to
  locally word-split $EDITOR on spaces in case it contains arguments.

- parse_file(): rewrite tag parsing to handle possible spaces in tags

- tags_in_post(): output line-separated instead of space-separated tags;
  further adjust sed script to handle possible spaces in tags

- rebuild_tags(): this function was refactored to use an array internally.
  Instead of two combined strings, it now takes HTML files and tags as
  separate arguments, separated by a single "--tag" argument. This allows
  for spaces and other special characters in both file names and tags. (See
  also commit a674ec5, which started this but didn't finish it).
This commit is contained in:
Martijn Dekker 2016-09-06 03:24:01 +02:00
parent d1a84d6eec
commit 54cc0c8060
1 changed files with 81 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

128
bb.sh
View file

@ -5,6 +5,18 @@
# https://github.com/carlesfe/bashblog/contributors
# Check out README.md for more details
# Some shell settings for robustness by default. These help eliminate
# unexpected snags and security vulnerabilities in case someone forgets to
# quote a variable somewhere. They do require a few coding adaptations.
IFS=$'\n' # Globally, we do word splitting only on newline (which also
# makes "$*" expand with newline separator instead of space).
set -f # Disable globbing (pathname expansion). It can be re-enabled
# locally using 'set +f'; it's handy to do this in a subshell,
# for example in $(command substitution), as the globbing will
# be local to the subshell.
# Global variables
# It is recommended to perform a 'rebuild' after changing any of this in the code
@ -252,6 +264,14 @@ get_html_file_content() {
}"
}
# Invoke the editor specified by the $EDITOR environment variable. Use a
# function for this as we need to locally word-split $EDITOR on spaces
# (in case it contains arguments, like EDITOR='joe -nobackups).
invoke_editor() {
local IFS=$' \t\n'
$EDITOR "$1"
}
# Edit an existing, published .html file while keeping its original timestamp
# Please note that this function does not automatically republish anything, as
# it is usually called from 'main'.
@ -270,7 +290,7 @@ edit() {
touch_timestamp=$(LC_ALL=C date -r "${1%%.*}.html" +'%Y%m%d%H%M')
tags_before=$(tags_in_post "${1%%.*}.html")
if [[ $2 == full ]]; then
$EDITOR "$1"
invoke_editor "$1"
filename=$1
else
if [[ ${1##*.} == md ]]; then
@ -280,7 +300,7 @@ edit() {
exit
fi
# editing markdown file
$EDITOR "$1"
invoke_editor "$1"
TMPFILE=$(markdown "$1")
filename=${1%%.*}.html
else
@ -290,7 +310,7 @@ edit() {
get_post_title "$1" > "$TMPFILE"
# Post text with plaintext tags
get_html_file_content 'text' 'text' <"$1" | sed "/^<p>$template_tags_line_header/s|<a href='$prefix_tags\([^']*\).html'>\\1</a>|\\1|g" >> "$TMPFILE"
$EDITOR "$TMPFILE"
invoke_editor "$TMPFILE"
filename=$1
fi
rm "$filename"
@ -306,10 +326,10 @@ edit() {
chmod 644 "$filename"
echo "Posted $filename"
tags_after=$(tags_in_post "$filename")
relevant_tags=$(echo "$tags_before $tags_after" | tr ',' ' ' | tr ' ' '\n' | sort -u | tr '\n' ' ')
if [[ ! -z $relevant_tags ]]; then
relevant_posts="$(posts_with_tags $relevant_tags) $filename"
rebuild_tags "$relevant_posts" "$relevant_tags"
relevant_tags=$(sort -u <<< "$tags_before"$'\n'"$tags_after")
if [[ -n $relevant_tags ]]; then
relevant_posts=$(posts_with_tags $relevant_tags)$'\n'$filename
rebuild_tags $relevant_posts --tags $relevant_tags
fi
}
@ -475,10 +495,11 @@ create_html_page() {
parse_file() {
# Read for the title and check that the filename is ok
title=""
while IFS='' read -r line; do
while read -r line; do
if [[ -z $title ]]; then
# remove extra <p> and </p> added by markdown
title=$(echo "$line" | sed 's/<\/*p>//g')
title=${line#<p>}
title=${title%</p>}
if [[ -n $3 ]]; then
filename=$3
else
@ -498,13 +519,14 @@ parse_file() {
content=$filename.tmp
# Parse possible tags
elif [[ $line == "<p>$template_tags_line_header"* ]]; then
tags=$(echo "$line" | cut -d ":" -f 2- | sed -e 's/<\/p>//g' -e 's/^ *//' -e 's/ *$//' -e 's/, /,/g')
IFS=, read -r -a array <<< "$tags"
echo -n "<p>$template_tags_line_header " >> "$content"
for item in "${array[@]}"; do
echo -n "<a href='$prefix_tags$item.html'>$item</a>, "
done | sed 's/, $/<\/p>/g' >> "$content"
sed "s%</p>%%g
s/^.*:[[:blank:]]*//
s/[[:blank:]]\$//
s/[[:blank:]]*,[[:blank:]]*/,/g
s%\([^,]*\),%<a href='$prefix_tags\1.html'>\1</a>, %g
s%, \([^,]*\)\$%, <a href='$prefix_tags\1.html'>\1</a></p>%
" <<< "$line" >> "$content"
else
echo "$line" >> "$content"
fi
@ -565,7 +587,7 @@ EOF
filename=""
while [[ $post_status != "p" && $post_status != "P" ]]; do
[[ -n $filename ]] && rm "$filename" # Delete the generated html file, if any
$EDITOR "$TMPFILE"
invoke_editor "$TMPFILE"
if [[ $fmt == md ]]; then
html_from_md=$(markdown "$TMPFILE")
parse_file "$html_from_md"
@ -607,8 +629,8 @@ EOF
echo "Posted $filename"
relevant_tags=$(tags_in_post $filename)
if [[ -n $relevant_tags ]]; then
relevant_posts="$(posts_with_tags $relevant_tags) $filename"
rebuild_tags "$relevant_posts" "$relevant_tags"
relevant_posts=$(posts_with_tags $relevant_tags)$'\n'$filename
rebuild_tags $relevant_posts --tags $relevant_tags
fi
}
@ -623,7 +645,7 @@ all_posts() {
{
echo "<h3>$template_archive_title</h3>"
prev_month=""
while IFS='' read -r i; do
for i in $(set +f; ls -t ./*.html); do
is_boilerplate_file "$i" && continue
echo -n "." 1>&3
# Month headers
@ -640,7 +662,7 @@ all_posts() {
# Date
date=$(LC_ALL=$date_locale date -r "$i" +"$date_format")
echo " $date</li>"
done < <(ls -t ./*.html)
done
echo "" 1>&3
echo "</ul>"
echo "<div id=\"all_posts\"><a href=\"./$index_file\">$template_archive_index_page</a></div>"
@ -663,7 +685,7 @@ all_tags() {
{
echo "<h3>$template_tags_title</h3>"
echo "<ul>"
for i in $prefix_tags*.html; do
for i in $(set +f; printf '%s\n' $prefix_tags*.html); do
[[ -f "$i" ]] || break
echo -n "." 1>&3
nposts=$(grep -c "<\!-- text begin -->" "$i")
@ -696,7 +718,8 @@ rebuild_index() {
# Create the content file
{
n=0
while IFS='' read -r i; do
for i in $(set +f; ls -t ./*.html) # sort by date, newest first
do
is_boilerplate_file "$i" && continue;
if ((n >= number_of_index_articles)); then break; fi
if [[ -n $cut_do ]]; then
@ -706,7 +729,7 @@ rebuild_index() {
fi
echo -n "." 1>&3
n=$(( n + 1 ))
done < <(ls -t ./*.html) # sort by date, newest first
done
feed=$blog_feed
if [[ -n $global_feedburner ]]; then feed=$global_feedburner; fi
@ -723,9 +746,18 @@ rebuild_index() {
# Finds all tags referenced in one post.
# Accepts either filename as first argument, or post content at stdin
# Prints one line with space-separated tags to stdout
# Prints tags to stdout, one per line.
# (Since we're doing global IFS word splitting on newline only,
# something like 'for tag in $(tags_in_post $i)' will work.)
tags_in_post() {
sed -n "/^<p>$template_tags_line_header/{s/^<p>$template_tags_line_header//;s/<[^>]*>//g;s/[ ,]\+/ /g;p;}" "$1" | tr ', ' ' '
local newline=$'\n'
sed -n "/^<p>$template_tags_line_header/ {
s/^<p>$template_tags_line_header[[:blank:]]*//
s/[[:blank:]]*<[^>]*>[[:blank:]]*//g
s/[[:blank:]]*,[[:blank:]]*/,/g
s/,\+/\\$newline/g
p
}" "$1"
}
# Finds all posts referenced in a number of tags.
@ -741,17 +773,15 @@ posts_with_tags() {
# Rebuilds tag_*.html files
# if no arguments given, rebuilds all of them
# if arguments given, they should have this format:
# "FILE1 [FILE2 [...]]" "TAG1 [TAG2 [...]]"
# FILE1 [FILE2 [...]] --tags TAG1 [TAG2 [...]]
# where FILEn are files with posts which should be used for rebuilding tags,
# and TAGn are names of tags which should be rebuilt.
# example:
# rebuild_tags "one_post.html another_article.html" "example-tag another-tag"
# mind the quotes!
# rebuild_tags one_post.html another_article.html --tags example-tag another-tag
rebuild_tags() {
local IFS=$'\n' # word splitting only on newline; make $* expand with newline as separator
if (($# < 1)); then
# will process all files and tags
files=( $(ls -t ./*.html) )
files=( $(set +f; ls -t ./*.html) )
all_tags=yes
else
# will process only given files and tags
@ -765,7 +795,7 @@ rebuild_tags() {
echo -n "Rebuilding tag pages "
n=0
if [[ -n $all_tags ]]; then
rm -f ./"$prefix_tags"*.html
( set +f; rm -f ./"$prefix_tags"*.html )
else
for i in "${tags[@]}"; do
rm -f "./$prefix_tags$i.html"
@ -792,12 +822,12 @@ rebuild_tags() {
done
rm "$tmpfile"
# Now generate the tag files with headers, footers, etc
while IFS='' read -r i; do
for i in $(set +f; ls -t ./"$prefix_tags"*.tmp.html 2>/dev/null); do
tagname=${i#./"$prefix_tags"}
tagname=${tagname%.tmp.html}
create_html_page "$i" "$prefix_tags$tagname.html" yes "$global_title &mdash; $template_tag_title \"$tagname\"" "$global_author"
rm "$i"
done < <(ls -t ./"$prefix_tags"*.tmp.html 2>/dev/null)
done
echo
}
@ -821,11 +851,12 @@ get_post_author() {
list_tags() {
if [[ $2 == -n ]]; then do_sort=1; else do_sort=0; fi
ls ./$prefix_tags*.html &> /dev/null
(($? != 0)) && echo "No posts yet. Use 'bb.sh post' to create one" && return
if ! (set +f; set -- $prefix_tags*.html; [[ -e $1 ]]); then
echo "No posts yet. Use 'bb.sh post' to create one"
return
fi
lines=""
for i in $prefix_tags*.html; do
for i in $(set +f; printf '%s\n' $prefix_tags*.html); do
[[ -f "$i" ]] || break
nposts=$(grep -c "<\!-- text begin -->" "$i")
tagname=${i#"$prefix_tags"}
@ -844,17 +875,19 @@ list_tags() {
# Displays a list of the posts
list_posts() {
ls ./*.html &> /dev/null
(($? != 0)) && echo "No posts yet. Use 'bb.sh post' to create one" && return
if ! (set +f; set -- *.html; [[ -e $1 ]]); then
echo "No posts yet. Use 'bb.sh post' to create one"
return
fi
lines=""
n=1
while IFS='' read -r i; do
for i in $(set +f; ls -t ./*.html); do
is_boilerplate_file "$i" && continue
line="$n # $(get_post_title "$i") # $(LC_ALL=$date_locale date -r "$i" +"$date_format")"
lines+=$line\\n
n=$(( n + 1 ))
done < <(ls -t ./*.html)
done
echo -e "$lines" | column -t -s "#"
}
@ -877,7 +910,7 @@ make_rss() {
echo "<atom:link href=\"$global_url/$blog_feed\" rel=\"self\" type=\"application/rss+xml\" />"
n=0
while IFS='' read -r i; do
for i in $(set +f; ls -t ./*.html); do
is_boilerplate_file "$i" && continue
((n >= number_of_feed_articles)) && break # max 10 items
echo -n "." 1>&3
@ -891,7 +924,7 @@ make_rss() {
echo "<pubDate>$(LC_ALL=C date -r "$i" +"%a, %d %b %Y %H:%M:%S %z")</pubDate></item>"
n=$(( n + 1 ))
done < <(ls -t ./*.html)
done
echo '</channel></rss>'
} 3>&1 >"$rssfile"
@ -989,7 +1022,8 @@ create_css() {
rebuild_all_entries() {
echo -n "Rebuilding all entries "
for i in ./*.html; do # no need to sort
for i in $(set +f; printf '%s\n' *.html) # no need to sort
do
is_boilerplate_file "$i" && continue;
contentfile=.tmp.$RANDOM
while [[ -f $contentfile ]]; do contentfile=.tmp.$RANDOM; done
@ -1042,7 +1076,7 @@ reset() {
echo "Are you sure you want to delete all blog entries? Please write \"Yes, I am!\" "
read -r line
if [[ $line == "Yes, I am!" ]]; then
rm .*.html ./*.html ./*.css ./*.rss &> /dev/null
(set +f; rm -f .*.html ./*.html ./*.css ./*.rss)
echo
echo "Deleted all posts, stylesheets and feeds."
echo "Kept your old '.backup.tar.gz' just in case, please delete it manually if needed."
@ -1114,9 +1148,9 @@ do_main() {
fi
# Test for existing html files
if ls ./*.html &> /dev/null; then
if (set +f; set -- *.html; [[ -e $1 ]]); then
# We're going to back up just in case
tar -c -z -f ".backup.tar.gz" -- *.html &&
(set +f; tar -c -z -f ".backup.tar.gz" -- *.html) &&
chmod 600 ".backup.tar.gz"
elif [[ $1 == rebuild ]]; then
echo "Can't find any html files, nothing to rebuild"