From 2151d814531c8e725b0e4886333deb12007c2a3e Mon Sep 17 00:00:00 2001 From: Michael Kroes Date: Tue, 18 Oct 2016 12:04:13 +0200 Subject: [PATCH] update to ownCloud 9.1.1 (with intermediate upgrades) (#894) [this is a squashed merge from-] * Install owncoud 9.1 and provide an upgrade path from 8.2. This also disables memcached and goes with apc. The upgrade fails with memcached. * Remove php apc setting * Add dav migrations for each user * Add some comments to the code * When upgrading owncloud from 8.2.3 to 9.1.0 the backup of 8.2.3 was overwritten when going from 9.0 to 9.1 * Add upgrade path from 8.1.1. Only do an upgrade check if owncloud was previously installed. * Stop php5-fpm before owncloud upgrade to prevent database locks * Fix fail2ban tests for owncloud 9 * When upgrading owncloud copy the database to the user-data/owncloud-backup directory * Remove not need unzip directives during owncloud extraction. Directory is removed beforehand so a normal extraction is fine * Improve backup of owncloud installation and provide a post installation restore script. Update the owncloud version number to 9.1.1. Update the calendar and contacts apps to the latest versions * Separate the ownCloud upgrades visually in the console output. --- setup/owncloud.sh | 117 ++++++++++++++++++++++++++++++-------- tests/fail2ban.py | 8 +-- tools/owncloud-restore.sh | 49 ++++++++++++++++ 3 files changed, 146 insertions(+), 28 deletions(-) create mode 100755 tools/owncloud-restore.sh diff --git a/setup/owncloud.sh b/setup/owncloud.sh index 2e590b0..deb46cb 100755 --- a/setup/owncloud.sh +++ b/setup/owncloud.sh @@ -16,10 +16,6 @@ apt_install \ apt-get purge -qq -y owncloud* -# Install ownCloud from source of this version: -owncloud_ver=8.2.7 -owncloud_hash=723ba3f46dad219109cdf28dcc016fcd8a6bc434 - # Migrate <= v0.10 setups that stored the ownCloud config.php in /usr/local rather than # in STORAGE_ROOT. Move the file to STORAGE_ROOT. if [ ! -f $STORAGE_ROOT/owncloud/config.php ] \ @@ -32,28 +28,34 @@ if [ ! -f $STORAGE_ROOT/owncloud/config.php ] \ ln -sf $STORAGE_ROOT/owncloud/config.php /usr/local/lib/owncloud/config/config.php fi -# Check if ownCloud dir exist, and check if version matches owncloud_ver (if either doesn't - install/upgrade) -if [ ! -d /usr/local/lib/owncloud/ ] \ - || ! grep -q $owncloud_ver /usr/local/lib/owncloud/version.php; then +InstallOwncloud() { + echo + echo "Upgrading to ownCloud version $1" + echo + + version=$1 + hash=$2 + + # Remove the current owncloud + rm -rf /usr/local/lib/owncloud # Download and verify - wget_verify https://download.owncloud.org/community/owncloud-$owncloud_ver.zip $owncloud_hash /tmp/owncloud.zip - - # Clear out the existing ownCloud. - if [ -d /usr/local/lib/owncloud/ ]; then - echo "upgrading ownCloud to $owncloud_ver (backing up existing ownCloud directory to /tmp/owncloud-backup-$$)..." - mv /usr/local/lib/owncloud /tmp/owncloud-backup-$$ - fi + wget_verify https://download.owncloud.org/community/owncloud-$version.zip $hash /tmp/owncloud.zip # Extract ownCloud - unzip -u -o -q /tmp/owncloud.zip -d /usr/local/lib #either extracts new or replaces current files + unzip -q /tmp/owncloud.zip -d /usr/local/lib rm -f /tmp/owncloud.zip - # The two apps we actually want are not in ownCloud core. Clone them from + # The two apps we actually want are not in ownCloud core. Download the releases from # their github repositories. mkdir -p /usr/local/lib/owncloud/apps - git_clone https://github.com/owncloudarchive/contacts 9ba2e667ae8c7ea36d8c4a4c3413c374beb24b1b '' /usr/local/lib/owncloud/apps/contacts - git_clone https://github.com/owncloudarchive/calendar 2086e738a3b7b868ec59cd61f0f88b49c3f21dd1 '' /usr/local/lib/owncloud/apps/calendar + wget_verify https://github.com/owncloud/contacts/releases/download/v1.4.0.0/contacts.tar.gz c1c22d29699456a45db447281682e8bc3f10e3e7 /tmp/contacts.tgz + tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/ + rm /tmp/contacts.tgz + + wget_verify https://github.com/nextcloud/calendar/releases/download/v1.4.0/calendar.tar.gz c84f3170efca2a99ea6254de34b0af3cb0b3a821 /tmp/calendar.tgz + tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/ + rm /tmp/calendar.tgz # Fix weird permissions. chmod 750 /usr/local/lib/owncloud/{apps,config} @@ -69,7 +71,7 @@ if [ ! -d /usr/local/lib/owncloud/ ] \ # If this isn't a new installation, immediately run the upgrade script. # Then check for success (0=ok and 3=no upgrade needed, both are success). - if [ -f $STORAGE_ROOT/owncloud/owncloud.db ]; then + if [ -e $STORAGE_ROOT/owncloud/owncloud.db ]; then # ownCloud 8.1.1 broke upgrades. It may fail on the first attempt, but # that can be OK. sudo -u www-data php /usr/local/lib/owncloud/occ upgrade @@ -81,6 +83,76 @@ if [ ! -d /usr/local/lib/owncloud/ ] \ echo "...which seemed to work." fi fi +} + +owncloud_ver=9.1.1 + +# Check if ownCloud dir exist, and check if version matches owncloud_ver (if either doesn't - install/upgrade) +if [ ! -d /usr/local/lib/owncloud/ ] \ + || ! grep -q $owncloud_ver /usr/local/lib/owncloud/version.php; then + + # Stop php-fpm + hide_output service php5-fpm stop + + # Backup the existing ownCloud. + # Create a backup directory to store the current installation and database to + BACKUP_DIRECTORY=$STORAGE_ROOT/owncloud-backup/`date +"%Y-%m-%d-%T"` + mkdir -p "$BACKUP_DIRECTORY" + if [ -d /usr/local/lib/owncloud/ ]; then + echo "upgrading ownCloud to $owncloud_ver (backing up existing ownCloud installation, configuration and database to directory to $BACKUP_DIRECTORY..." + cp -r /usr/local/lib/owncloud "$BACKUP_DIRECTORY/owncloud-install" + fi + if [ -e /home/user-data/owncloud/owncloud.db ]; then + cp /home/user-data/owncloud/owncloud.db $BACKUP_DIRECTORY + fi + if [ -e /home/user-data/owncloud/config.php ]; then + cp /home/user-data/owncloud/config.php $BACKUP_DIRECTORY + fi + + # We only need to check if we do upgrades when owncloud was previously installed + if [ -e /usr/local/lib/owncloud/version.php ]; then + if grep -q "8.1.[0-9]" /usr/local/lib/owncloud/version.php; then + echo "We are running 8.1.x, upgrading to 8.2.3 first" + InstallOwncloud 8.2.3 bfdf6166fbf6fc5438dc358600e7239d1c970613 + fi + + # If we are upgrading from 8.2.x we should go to 9.0 first. Owncloud doesn't support skipping minor versions + if grep -q "8.2.[0-9]" /usr/local/lib/owncloud/version.php; then + echo "We are running version 8.2.x, upgrading to 9.0.2 first" + + # We need to disable memcached. The upgrade and install fails + # with memcached + CONFIG_TEMP=$(/bin/mktemp) + php < $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php; + +EOF + chown www-data.www-data $STORAGE_ROOT/owncloud/config.php + + # We can now install owncloud 9.0.2 + InstallOwncloud 9.0.2 72a3d15d09f58c06fa8bee48b9e60c9cd356f9c5 + + # The owncloud 9 migration doesn't migrate calendars and contacts + # The option to migrate these are removed in 9.1 + # So the migrations should be done when we have 9.0 installed + sudo -u www-data php /usr/local/lib/owncloud/occ dav:migrate-addressbooks + # The following migration has to be done for each owncloud user + for directory in $STORAGE_ROOT/owncloud/*@*/ ; do + username=$(basename "${directory}") + sudo -u www-data php /usr/local/lib/owncloud/occ dav:migrate-calendar $username + done + sudo -u www-data php /usr/local/lib/owncloud/occ dav:sync-birthday-calendar + fi + fi + + InstallOwncloud $owncloud_ver 72ed9812432f01b3a459c4afc33f5c76b71eec09 fi # ### Configuring ownCloud @@ -110,10 +182,7 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then 'arguments'=>array('{127.0.0.1:993/imap/ssl/novalidate-cert}') ) ), - 'memcache.local' => '\\OC\\Memcache\\Memcached', - "memcached_servers" => array ( - array('127.0.0.1', 11211), - ), + 'memcache.local' => '\OC\Memcache\APC', 'mail_smtpmode' => 'sendmail', 'mail_smtpsecure' => '', 'mail_smtpauthtype' => 'LOGIN', @@ -173,7 +242,7 @@ include("$STORAGE_ROOT/owncloud/config.php"); \$CONFIG['trusted_domains'] = array('$PRIMARY_HOSTNAME'); -\$CONFIG['memcache.local'] = '\\OC\\Memcache\\Memcached'; +\$CONFIG['memcache.local'] = '\OC\Memcache\APC'; \$CONFIG['overwrite.cli.url'] = '/cloud'; \$CONFIG['mail_from_address'] = 'administrator'; # just the local part, matches our master administrator address diff --git a/tests/fail2ban.py b/tests/fail2ban.py index ec9530f..0a3c1da 100644 --- a/tests/fail2ban.py +++ b/tests/fail2ban.py @@ -10,11 +10,11 @@ import sys, os, time, functools # parse command line -if len(sys.argv) != 3: - print("Usage: tests/fail2ban.py \"ssh user@hostname\" hostname") +if len(sys.argv) != 4: + print("Usage: tests/fail2ban.py \"ssh user@hostname\" hostname owncloud_user") sys.exit(1) -ssh_command, hostname = sys.argv[1:3] +ssh_command, hostname, owncloud_user = sys.argv[1:4] # define some test types @@ -215,7 +215,7 @@ if __name__ == "__main__": run_test(http_test, ["/admin/munin/", 401], 20, 30, 1) # ownCloud - run_test(http_test, ["/cloud/remote.php/webdav", 401, None, None, ["aa", "aa"]], 20, 120, 1) + run_test(http_test, ["/cloud/remote.php/webdav", 401, None, None, [owncloud_user, "aa"]], 20, 120, 1) # restart fail2ban so that this client machine is no longer blocked restart_fail2ban_service(final=True) diff --git a/tools/owncloud-restore.sh b/tools/owncloud-restore.sh new file mode 100755 index 0000000..0e6a075 --- /dev/null +++ b/tools/owncloud-restore.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# This script will restore the backup made during an installation +source /etc/mailinabox.conf # load global vars + +if [ -z "$1" ]; then + echo "Usage: owncloud-restore.sh " + echo + echo "WARNING: This will restore the database to the point of the installation!" + echo " This means that you will lose all changes made by users after that point" + echo + echo + echo "Backups are stored here: $STORAGE_ROOT/owncloud-backup/" + echo + echo "Available backups:" + echo + find $STORAGE_ROOT/owncloud-backup/* -maxdepth 0 -type d + echo + echo "Supply the directory that was created during the last installation as the only commandline argument" + exit +fi + +if [ ! -f $1/config.php ]; then + echo "This isn't a valid backup location" + exit +fi + +echo "Restoring backup from $1" +service php5-fpm stop + +# remove the current owncloud installation +rm -rf /usr/local/lib/owncloud/ +# restore the current owncloud application +cp -r "$1/owncloud-install" /usr/local/lib/owncloud + +# restore access rights +chmod 750 /usr/local/lib/owncloud/{apps,config} + +cp "$1/owncloud.db" $STORAGE_ROOT/owncloud/ +cp "$1/config.php" $STORAGE_ROOT/owncloud/ + +ln -sf $STORAGE_ROOT/owncloud/config.php /usr/local/lib/owncloud/config/config.php +chown -f -R www-data.www-data $STORAGE_ROOT/owncloud /usr/local/lib/owncloud +chown www-data.www-data $STORAGE_ROOT/owncloud/config.php + +sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off + +service php5-fpm start +echo "Done"