dynamicbrain.neuroinf.jp/etc/dumpMediaWiki.php

<?php

define('PROTECTOR_SKIP_DOS_CHECK', 1);
define('PROTECTOR_SKIP_FILESCHECKER', 1);
define('XOOPS_LOGGER_ADDQUERY_DISABLED', true);
$_SERVER['HTTP_USER_AGENT'] = 'php-cli';
//$_SERVER['REQUEST_METHOD'] = 'GET';
$_ENV['HTTP_REFERER'] = 'http://localhost/index.php';
$_SERVER['QUERY_STRING'] = '/index.php';
$_SERVER['REMOTE_ADDR'] = '192.168.0.1';

$basedir = '/data/www/html/modules/mediawiki/w/maintenance';

define('API_URL', 'https://dynamicbrain.neuroinf.jp/modules/mediawiki/w/api.php');
define('IMAGE_DIR', '/data/www/html/uploads/mediawiki');
define('DEST_IMAGE_DIR', __DIR__.'/data/public/mediawiki/images');

require_once $basedir.'/Maintenance.php';

class DumpMediaWiki extends Maintenance
{
    public function execute()
    {
        error_reporting(E_ALL);
        while (ob_get_level()) {
            ob_end_clean();
        }
        $mwlist = [];
        $this->makedir(__DIR__.'/data');
        $this->makedir(__DIR__.'/data/src');
        $this->makedir(__DIR__.'/data/src/mediawiki');
        $this->makedir(__DIR__.'/data/src/mediawiki/assets');
        $this->makedir(__DIR__.'/data/public');
        $this->makedir(__DIR__.'/data/public/mediawiki');
        $this->makedir(__DIR__.'/data/public/mediawiki/images');
        $this->makedir(__DIR__.'/data/public/mediawiki/contents');
        $this->makedir(__DIR__.'/temp');
        $this->makedir(__DIR__.'/temp/mediawiki');
        $url = API_URL.'?action=query&list=allpages&aplimit=500&utf8=&format=json';
        $json = $this->fetchData($url);
        $data = json_decode($json, true);
        $allpages = $data['query']['allpages'];
        $num = 0;
        $start = 0;
        foreach ($allpages as $page) {
            if ($num < $start) {
                ++$num;
                continue;
            }
            $name = urlencode(str_replace(' ', '_', $page['title']));
            $url = API_URL.'?action=parse&format=json&page='.$name;
            $json = $this->fetchData($url);
            $data = json_decode($json, true);
            $detail = $data['parse'];
            echo $num.'('.$detail['revid'].'): '.$page['title'].PHP_EOL;
            $mwlist[] = [
                'id' => (int) $detail['revid'],
                'title' => $page['title'],
            ];
            if (!isset($detail['images'])) {
                var_dump($json);
                var_dump($page);
                var_dump($name);
                var_dump($data);
                exit();
            }
            foreach ($detail['images'] as $image) {
                $fpath = $this->getImagePath(IMAGE_DIR, $image);
                if (!file_exists($fpath)) {
                    exit('File not found: '.$fpath.PHP_EOL);
                }
                $dfpath = $this->getImagePath(DEST_IMAGE_DIR, $image);
                $pddpath = dirname($dfpath);
                $ppddpath = dirname($pddpath);
                $this->makedir($ppddpath);
                $this->makedir($pddpath);
                $this->fileCopy($fpath, $dfpath);
            }
            $text = $this->fixHtml($detail['text']['*']);
            $title = $detail['title'];
            $sections = [];
            foreach ($detail['sections'] as $section) {
                $sections[] = [
                    'toclevel' => $section['toclevel'],
                    'line' => $section['line'],
                    'number' => $section['number'],
                    'anchor' => $section['anchor'],
                ];
            }
            $data = [
               'title' => $title,
               'text' => $text,
               'sections' => $sections,
            ];
            $fname = __DIR__.'/temp/mediawiki/'.$detail['revid'].'.dump.json';
            file_put_contents($fname, json_encode($data, JSON_UNESCAPED_UNICODE));
            ++$num;
        }
        $fname = __DIR__.'/data/src/mediawiki/assets/contents.json';
        file_put_contents($fname, json_encode($mwlist, JSON_UNESCAPED_UNICODE));

        echo "Hello, World!\n";
    }

    public function fixHtml($text)
    {
        $text = preg_replace('/<span class="mw-editsection"><span class="mw-editsection-bracket">\[<\/span><a href="(?:[^"]*)" title="(?:[^"]*)">edit<\/a><span class="mw-editsection-bracket">\]<\/span><\/span>/Us', '', $text);
        $text = preg_replace('/<a href="\/modules\/mediawiki\/w\/index.php\?title=(?:[^"]+)&amp;action=edit&amp;redlink=1" class="new" title="(?:[^"]*)">([^<]+)<\/a>/Us', '\1', $text);
        $text = preg_replace('/<li>\s*(.*)\s*<\/li>/Us', '<li>\1</li>', $text);
        $text = preg_replace('/<!-- .* -->/Us', '', $text);
        $text = trim($text);

        return $text;
    }

    public function makedir($path)
    {
        if (is_dir($path)) {
            return true;
        }
        if (false === @mkdir($path)) {
            exit('Failed to create directroy: '.$path.PHP_EOL);
        }

        return true;
    }

    public function getImagePath($basedir, $fname)
    {
        $hash = md5($fname);

        return $basedir.'/'.substr($hash, 0, 1).'/'.substr($hash, 0, 2).'/'.$fname;
    }

    public function fileCopy($from, $to)
    {
        system('rsync -aq '.escapeshellarg($from).' '.escapeshellarg($to), $ret);
        if (0 !== $ret) {
            exit('Failed to copy file: '.$from.' -> '.$to.PHP_EOL);
        }

        return 0 === $ret;
    }

    public function fetchData($url)
    {
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_URL, $url);
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($curl, CURLOPT_AUTOREFERER, true);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($curl, CURLOPT_FAILONERROR, true);
        curl_setopt($curl, CURLOPT_TIMEOUT, 5);
        $ret = curl_exec($curl);
        if (false === $ret) {
            exit('Failed to fetch data, unexpected error: '.$url.PHP_EOL);
        }
        $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
        if (200 !== $code) {
            exit('Failed to fetch data, invalid status code('.$code.'): '.$url.PHP_EOL);
        }
        curl_close($curl);

        return $ret;
    }

    public function fileDownload($url, $fpath)
    {
        $fp = fopen($fpath, 'w');
        if (false === $fp) {
            exit('Failed to open file: '.$fpath.PHP_EOL);
        }
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_URL, $url);
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($curl, CURLOPT_AUTOREFERER, true);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($curl, CURLOPT_FAILONERROR, true);
        curl_setopt($curl, CURLOPT_FILE, $fp);
        curl_setopt($curl, CURLOPT_TIMEOUT, 5);
        $ret = curl_exec($curl);
        fclose($fp);
        if (false === $ret) {
            exit('Failed to download file, unexpected error: '.$url.PHP_EOL);
        }
        $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
        if (200 !== $code) {
            exit('Failed to download file, invalid status code('.$code.'): '.$url.PHP_EOL);
        }
        curl_close($curl);

        return true;
    }
}

$maintClass = DumpMediaWiki::class;

require_once RUN_MAINTENANCE_IF_MAIN;