ruby/tool/make-snapshot
Nobuyoshi Nakada 9645d193dd [Misc #22206] Fix snapshot dependency generation
Skip dependency generation when exporting stable branches that
predate the Ruby dependency scanner.  Avoid loading GNU make
definitions into the temporary Makefile so snapshot generation also
works with BSD make.
2026-07-23 11:38:55 +09:00

677 lines
19 KiB
Ruby
Executable File

#!/usr/bin/ruby -s
# -*- coding: us-ascii -*-
require 'rubygems'
require 'rubygems/package'
require 'rubygems/package/tar_writer'
require 'uri'
require 'digest/sha1'
require 'digest/sha2'
require 'fileutils'
require 'shellwords'
require 'tmpdir'
require 'pathname'
require 'date'
require 'yaml'
require 'json'
require File.expand_path("../lib/vcs", __FILE__)
require File.expand_path("../lib/colorize", __FILE__)
STDOUT.sync = true
$srcdir ||= nil
$archname = nil if ($archname ||= nil) == ""
$keep_temp ||= nil
$patch_file ||= nil
$packages ||= nil
$digests ||= nil
$no7z ||= nil
$tooldir = File.expand_path("..", __FILE__)
$unicode_version = nil if ($unicode_version ||= nil) == ""
$colorize = Colorize.new
def usage
<<USAGE
usage: #{File.basename $0} [option...] new-directory-to-save [version ...]
options:
-srcdir=PATH source directory path
-archname=NAME make the basename of snapshots NAME
-keep_temp keep temporary working directory
-patch_file=PATCH apply PATCH file after export
-packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")})
-digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")})
-unicode_version=VER Unicode version to generate encodings
-help, --help show this message
version:
master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL
each versions may be followed by optional @revision.
USAGE
end
DIGESTS = %w[SHA1 SHA256 SHA512]
PACKAGES = {
"tar" => %w".tar",
"bzip" => %w".tar.bz2 bzip2 -c",
"gzip" => %w".tar.gz gzip -c",
"xz" => %w".tar.xz xz -c",
"zip" => %w".zip zip -Xqr",
}
DEFAULT_PACKAGES = PACKAGES.keys - ["tar", "bzip"]
if !$no7z and system("7z", out: IO::NULL)
PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so"
PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL}
elsif gzip = ENV.delete("GZIP")
PACKAGES["gzip"].concat(gzip.shellsplit)
end
if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags
mflags = mflags.gsub(/(\A|\s)(-\S*)j\d*/, '\1\2')
mflags.strip!
ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?)
end
ENV["LC_ALL"] = ENV["LANG"] = "C"
# https git clone is disabled at git.ruby-lang.org/ruby.git.
GITURL = URI.parse("https://github.com/ruby/ruby.git")
RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/
ENV["TZ"] = "UTC"
class String
# for older ruby
alias bytesize size unless method_defined?(:bytesize)
end
class Dir
def self.mktmpdir(path)
path = File.join(tmpdir, path+"-#{$$}-#{rand(100000)}")
begin
mkdir(path)
rescue Errno::EEXIST
path.succ!
retry
end
path
end unless respond_to?(:mktmpdir)
end
$packages &&= $packages.split(/[, ]+/).tap {|pkg|
if all = pkg.index("all")
pkg[all, 1] = DEFAULT_PACKAGES - pkg
end
pkg -= PACKAGES.keys
pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}"
}
$packages ||= DEFAULT_PACKAGES
$digests &&= $digests.split(/[, ]+/).tap {|dig|
dig -= DIGESTS
dig.empty? or abort "#{File.basename $0}: unknown digests - #{dig.join(", ")}"
}
$digests ||= DIGESTS
$patch_file &&= File.expand_path($patch_file)
PATH = ENV["PATH"].split(File::PATH_SEPARATOR)
def PATH.executable_env(var, command = nil)
command = ENV[var] ||= (command or return)
cmd, = command.shellsplit
unless any? {|dir|
file = File.expand_path(cmd, dir)
File.file?(file) and File.executable?(file)
}
abort "#{File.basename $0}: #{var} command not found - #{cmd}"
end
command
end
PATH.executable_env("MV", "mv")
PATH.executable_env("RM", "rm -f")
PATH.executable_env("AUTOCONF", "autoconf")
%w[BASERUBY RUBY MINIRUBY].each do |var|
cmd = PATH.executable_env(var, "ruby")
help = IO.popen("#{cmd} --help", err: %i[child out], &:read)
unless $?.success? and /ruby/ =~ help
abort "#{File.basename $0}: #{var} ruby not found - #{cmd}"
end
IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read)
cmd += ' --disable-gem' if $?.success?
ENV[var] = cmd
end
if defined?($help) or defined?($_help)
puts usage
exit
end
unless destdir = ARGV.shift
abort usage
end
revisions = ARGV.empty? ? [nil] : ARGV
if defined?($exported)
abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead"
end
FileUtils.mkpath(destdir)
destdir = File.expand_path(destdir)
tmp = Dir.mktmpdir("ruby-snapshot")
FileUtils.mkpath(tmp)
at_exit {
Dir.chdir "/"
FileUtils.rm_rf(tmp)
} unless $keep_temp
def tar_create(tarball, dir)
header = Gem::Package::TarHeader
dir_type = "5"
uname = gname = "ruby"
File.open(tarball, "wb") do |f|
w = Gem::Package::TarWriter.new(f)
list = Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH)
list.reject! {|name| name.end_with?("/.")}
list.sort_by! {|name| name.split("/")}
list.each do |path|
next if File.basename(path) == "."
s = File.stat(path)
mode = 0644
case
when s.file?
type = nil
size = s.size
mode |= 0111 if s.executable?
when s.directory?
path += "/"
type = dir_type
size = 0
mode |= 0111
else
next
end
name, prefix = w.split_name(path)
h = header.new(name: name, prefix: prefix, typeflag: type,
mode: mode, size: size, mtime: s.mtime,
uname: uname, gname: gname)
f.write(h)
if size > 0
IO.copy_stream(path, f)
f.write("\0" * (-size % 512))
end
end
end
true
rescue => e
warn e.message
false
end
def touch_all(time, pattern, opt, &cond)
Dir.glob(pattern, opt) do |n|
stat = File.stat(n)
if stat.file? or stat.directory?
next if cond and !yield(n, stat)
File.utime(time, time, n)
end
end
rescue
false
else
true
end
class MAKE < Struct.new(:prog, :args)
def initialize(vars)
vars = vars.map {|arg| arg.join("=")}
super(ENV["MAKE"] || ENV["make"] || "make", vars)
end
def run(target)
err = IO.pipe do |r, w|
begin
pid = Process.spawn(self.prog, *self.args, target, {:err => w, r => :close})
w.close
r.read
ensure
Process.wait(pid)
end
end
if $?.success?
true
else
STDERR.puts err
$colorize.fail("#{target} failed")
false
end
end
end
def measure
clock = Process::CLOCK_MONOTONIC
t0 = Process.clock_gettime(clock)
STDOUT.flush
result = yield
printf(" %6.3f", Process.clock_gettime(clock) - t0)
STDOUT.flush
result
end
def package(vcs, rev, destdir, tmp = nil)
pwd = Dir.pwd
prerelease = false
if rev and revision = rev[/@(\h+)\z/, 1]
rev = $`
end
case rev
when nil
url = nil
when /\A(?:master|trunk)\z/
url = vcs.trunk
when /\Abranches\//
url = vcs.branch($')
when /\Atags\//
url = vcs.tag($')
when /\Astable\z/
vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]}
url &&= vcs.branch(url)
when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/
prerelease = true
tag = "#{$4}#{$5}"
if Integer($1) >= 4
url = vcs.tag("v#{rev}")
else
url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}")
end
when /\A(\d+)\.(\d+)\.(\d+)\z/
tag = ""
if Integer($1) >= 4
url = vcs.tag("v#{rev}")
else
url = vcs.tag("v#{$1}_#{$2}_#{$3}")
end
when /\A(\d+)\.(\d+)\z/
url = vcs.branch("ruby_#{rev.tr('.', '_')}")
else
warn "#{$0}: unknown version - #{rev}"
return
end
if info = vcs.get_revisions(url)
modified = info[2]
else
_, _, modified = VCS::Null.new(nil).get_revisions(url)
end
if !revision and info
revision = info
url ||= vcs.branch(revision[3])
revision = revision[1]
end
version = nil
unless revision
url = vcs.trunk
vcs.grep(RUBY_VERSION_PATTERN, url, "version.h") {version = $1}
unless rev == version
warn "#{$0}: #{rev} not found"
return
end
revision = vcs.get_revisions(url)[1]
end
v = "ruby"
puts "Exporting #{rev}@#{revision}"
exported = tmp ? File.join(tmp, v) : v
unless vcs.export(revision, url, exported, true) {|line| print line}
warn("Export failed")
return
end
if $srcdir
Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file|
puts "copying #{file}" if $VERBOSE
dest = exported + file[$srcdir.size..-1]
FileUtils.mkpath(File.dirname(dest))
begin
FileUtils.cp_r(file, dest)
FileUtils.chmod_R("a+rwX,go-w", dest)
rescue SystemCallError
end
end
end
Dir.glob("#{exported}/.*.yml") do |file|
FileUtils.rm(file, verbose: $VERBOSE)
end
status = File.read(File.dirname(__FILE__) + "/prereq.status")
Dir.chdir(tmp) if tmp
if !File.directory?(v)
v = Dir.glob("ruby-*").select(&File.method(:directory?))
v.size == 1 or abort "#{File.basename $0}: not exported"
v = v[0]
end
File.open("#{v}/revision.h", "wb") {|f|
f.puts vcs.revision_header(revision, modified)
}
version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1]
version ||=
begin
include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h")
api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1]
api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1]
version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1]
[api_major_version, api_minor_version, version_teeny].join('.')
end
version or return
if prerelease
versionhdr ||= File.read("#{v}/version.h")
versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace"
File.write("#{v}/version.h", versionhdr)
else
tag ||= vcs.revision_name(revision)
end
if $archname
n = $archname
elsif tag.empty?
n = "ruby-#{version}"
else
n = "ruby-#{version}-#{tag}"
end
File.directory?(n) or File.rename v, n
v = n
if $patch_file && !system(*%W"patch -d #{v} -p0 -i #{$patch_file}")
puts $colorize.fail("patching failed")
return
end
class << (clean = [])
def add(n) push(n)
n
end
def create(file, content = "", &block)
add(file)
if block
File.open(file, "wb", &block)
else
File.binwrite(file, content)
end
end
end
Dir.chdir(v) do
unless File.exist?("ChangeLog")
vcs.export_changelog(url, nil, revision, "ChangeLog")
end
unless touch_all(modified, "**/*", File::FNM_DOTMATCH)
modified = nil
colors = %w[red yellow green cyan blue magenta]
"take a breath, and go ahead".scan(/./) do |c|
if c == ' '
print c
else
colors.push(color = colors.shift)
print $colorize.decorate(c, color)
end
sleep(c == "," ? 0.7 : 0.05)
end
puts
end
clean.create("cross.rb") do |f|
f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)"
f.puts "CROSS_COMPILING=true"
f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)"
f.puts "RUBY_PLATFORM='none'"
f.puts "Object.__send__(:remove_const, :RUBY_VERSION)"
f.puts "RUBY_VERSION='#{version}'"
end
puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE
unless File.exist?("configure")
print "creating configure..."
unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2)
puts $colorize.fail(" failed")
return
end
puts $colorize.pass(" done")
end
clean.add("autom4te.cache")
clean.add("enc/unicode/data")
print "creating prerequisites..."
if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk")
puts
extout = clean.add('tmp')
begin
status = File.read("tool/prereq.status")
rescue Errno::ENOENT
# use fallback file
end
clean.create("config.status", status)
clean.create("noarch-fake.rb", "require_relative 'cross'\n")
FileUtils.mkpath(hdrdir = "#{extout}/include/ruby")
File.binwrite("#{hdrdir}/config.h", "")
FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults")
File.binwrite("#{defaults}/operating_system.rb", "")
File.binwrite("#{defaults}/ruby.rb", "")
miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross"
baseruby = ENV["BASERUBY"]
mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")).
gsub(/^@.*\n/, '')
vars = {
"EXTOUT"=>extout,
"PATH_SEPARATOR"=>File::PATH_SEPARATOR,
"MINIRUBY"=>miniruby,
"RUBY"=>ENV["RUBY"],
"BASERUBY"=>baseruby,
"PWD"=>Dir.pwd,
"ruby_version"=>version,
"MAJOR"=>api_major_version,
"MINOR"=>api_minor_version,
"TEENY"=>version_teeny,
"VPATH"=>(ENV["VPATH"] || "include/ruby"),
"PROGRAM"=>(ENV["PROGRAM"] || "ruby"),
"BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"),
"DUMP_AST"=>"build-tool/dump_ast#{RbConfig::CONFIG['EXEEXT']} ",
}
status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do
vars[$2] ||= $3
end
vars.delete("UNICODE_FILES") # for stable branches
vars["UNICODE_VERSION"] = $unicode_version if $unicode_version
args = vars.dup
args["MKDEPEND_FILES"] = "--scope=core"
args["MKDEPEND_OPTIONS"] = ""
mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]}
commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do
if inc = $1 and File.exist?(inc)
File.binread(inc).gsub(/^!/, '# !')
else
"#"
end
end
mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile')
mk << <<-'APPEND'
update-download:: touch-unicode-files
prepare-package: prereq after-update
clean-cache: $(CLEAN_CACHE)
after-update:: extract-gems
extract-gems: update-gems
update-gems:
$(UNICODE_SRC_DATA_DIR)/.unicode-tables.time:
touch-unicode-files:
APPEND
clean.create("Makefile", mk)
clean.create("revision.tmp")
clean.create(".revision.time")
ENV["CACHE_SAVE"] = "no"
make = MAKE.new(args)
if File.file?("tool/mkdepend.rb")
return unless make.run("fix-depends")
end
return unless make.run("update-download")
clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk", "ext/ripper/y.output", ".revision.time")
Dir.glob("**/*") do |dest|
next unless File.symlink?(dest)
orig = File.expand_path(File.readlink(dest), File.dirname(dest))
File.unlink(dest)
FileUtils.cp_r(orig, dest)
end
File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"]))
return unless make.run("prepare-package")
return unless make.run("clean-cache")
return unless make.run("clean")
if modified
new_time = modified + 2
touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat|
stat.mtime > modified unless clean.include?(name)
end
modified = new_time
end
print "prerequisites"
else
system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y])
end
vcs.after_export(".") if exported
clean.concat(Dir.glob("ext/**/autom4te.cache"))
clean.add(".downloaded-cache")
if File.exist?("gems/bundled_gems")
gems = Dir.glob("gems/*.gem")
gems -= File.readlines("gems/bundled_gems").map {|line|
next if /^\s*(?:#|$)/ =~ line
name, version, _ = line.split(' ')
"gems/#{name}-#{version}.gem"
}
clean.concat(gems)
else
clean.add("gems")
end
FileUtils.rm_rf(clean)
if modified
touch_all(modified, "**/*/", 0) do |name, stat|
stat.mtime > modified
end
File.utime(modified, modified, ".")
end
unless $?.success?
puts $colorize.fail(" failed")
return
end
puts $colorize.pass(" done")
end
if v == "."
v = File.basename(Dir.pwd)
Dir.chdir ".."
else
Dir.chdir(File.dirname(v))
v = File.basename(v)
end
tarball = nil
return $packages.collect do |mesg|
(ext, *cmd) = PACKAGES[mesg]
File.directory?(destdir) or FileUtils.mkpath(destdir)
file = File.join(destdir, "#{$archname||v}#{ext}")
case ext
when /\.tar/
if tarball
next if tarball.empty?
else
tarball = ext == ".tar" ? file : "#{$archname||v}.tar"
print "creating tarball... #{tarball}"
if measure {tar_create(tarball, v)}
puts $colorize.pass(" done")
File.utime(modified, modified, tarball) if modified
next if tarball == file
else
puts $colorize.fail(" failed")
tarball = ""
next
end
end
print "creating #{mesg} tarball... #{file}"
done = measure {system(*cmd, tarball, out: file)}
else
print "creating #{mesg} archive... #{file}"
if Hash === cmd.last
*cmd, opt = *cmd
cmd << file << v << opt
else
(cmd = cmd.dup) << file << v
end
done = measure {system(*cmd)}
end
if done
puts $colorize.pass(" done")
file
else
puts $colorize.fail(" failed")
nil
end
end.compact
ensure
FileUtils.rm_rf(tmp ? File.join(tmp, v) : v) if v and !$keep_temp
Dir.chdir(pwd)
end
if [$srcdir, ($git||=nil)].compact.size > 1
abort "#{File.basename $0}: -srcdir and -git are exclusive"
end
if $srcdir
vcs = VCS.detect($srcdir)
elsif $git
abort "#{File.basename $0}: use -srcdir with cloned local repository"
else
begin
vcs = VCS.detect(File.expand_path("../..", __FILE__))
rescue VCS::NotFoundError
abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository"
end
end
release_date = Time.now.getutc
info = {}
success = true
revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name|
if !name
success = false
next
end
str = File.binread(name)
pathname = Pathname(name)
basename = pathname.basename.to_s
extname = pathname.extname.sub(/\A\./, '')
version = basename[/\Aruby-(.*)\.(?:tar|zip)/, 1]
key = basename[/\A(.*)\.(?:tar|zip)/, 1]
info[key] ||= Hash.new{|h,k|h[k]={}}
info[key]['version'] = version if version
info[key]['date'] = release_date.to_date
if version
info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/"
info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}"
else
info[key]['filename'][extname] = basename
end
info[key]['size'][extname] = str.bytesize
puts "* #{$colorize.pass(name)}"
puts " SIZE: #{str.bytesize} bytes"
$digests.each do |alg|
digest = Digest(alg).hexdigest(str)
info[key][alg.downcase][extname] = digest
printf " %-8s%s\n", "#{alg}:", digest
end
end
yaml = info.values.to_yaml
json = info.values.to_json
puts "#{$colorize.pass('YAML:')}"
puts yaml
puts "#{$colorize.pass('JSON:')}"
puts json
infodir = Pathname(destdir) + 'info'
infodir.mkpath
(infodir+'info.yml').write(yaml)
(infodir+'info.json').write(json)
exit false if !success
# vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix