fix: crash when exiting linux listen server (#241)
fix: crash when loading on windows xp (#244) build: switched to github hosted runners, and get rid of self-hosted build: windows exe and dll is now code-signed build: drop support for intel icc compiler
This commit is contained in:
dmitry
parent
c5bdc8dfec
commit
1e9bc3cb5f
9 changed files with 456 additions and 287 deletions
326
package.py
326
package.py
|
|
@ -6,127 +6,223 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
|
||||
import os, sys, subprocess
|
||||
from genericpath import isdir
|
||||
import os, sys, subprocess, base64
|
||||
import locale, urllib3
|
||||
import pathlib, shutil
|
||||
import zipfile, tarfile
|
||||
import datetime, calendar
|
||||
|
||||
from github import Github
|
||||
class CodeSign (object):
|
||||
def __init__(self, product, url, verify_signature=False):
|
||||
self.signing = True
|
||||
|
||||
self.ossl_path = "/usr/bin/osslsigncode"
|
||||
self.local_key = os.path.join (pathlib.Path ().absolute (), "key.pfx");
|
||||
|
||||
self.product = product
|
||||
self.url = url
|
||||
self.verify_signature = verify_signature
|
||||
|
||||
if not os.path.exists (self.ossl_path):
|
||||
self.signing = False
|
||||
|
||||
if not "CS_CERTIFICATE" in os.environ:
|
||||
self.signing = False
|
||||
|
||||
if not "CS_CERTIFICATE_PASSWORD" in os.environ:
|
||||
self.signing = False
|
||||
|
||||
if self.signing:
|
||||
self.password = os.environ.get ("CS_CERTIFICATE_PASSWORD")
|
||||
|
||||
encoded = os.environ.get ("CS_CERTIFICATE")
|
||||
|
||||
if len (encoded) < 64:
|
||||
print ('Damaged certificate. Signing disabled.')
|
||||
self.signing = False
|
||||
return
|
||||
|
||||
decoded = base64.b64decode (encoded)
|
||||
|
||||
with open (self.local_key, "wb") as key:
|
||||
key.write (decoded)
|
||||
|
||||
def has(self):
|
||||
return self.signing
|
||||
|
||||
def sign_file_inplace (self, filename):
|
||||
signed_filename = filename + ".signed"
|
||||
sign = []
|
||||
|
||||
sign.append (self.ossl_path)
|
||||
sign.append ("sign")
|
||||
sign.append ("-pkcs12")
|
||||
sign.append (self.local_key)
|
||||
sign.append ("-pass")
|
||||
sign.append (self.password)
|
||||
sign.append ("-n")
|
||||
sign.append (self.product)
|
||||
sign.append ("-i")
|
||||
sign.append (self.url)
|
||||
sign.append ("-h")
|
||||
sign.append ("sha256")
|
||||
sign.append ("-t")
|
||||
sign.append ("http://timestamp.sectigo.com")
|
||||
sign.append ("-in")
|
||||
sign.append (filename)
|
||||
sign.append ("-out")
|
||||
sign.append (signed_filename)
|
||||
|
||||
result = subprocess.run (sign, capture_output=True, text=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
os.unlink (filename)
|
||||
shutil.move (signed_filename, filename)
|
||||
|
||||
print ("Signing result: {}".format (result.stdout))
|
||||
|
||||
if self.verify_signature:
|
||||
verify = []
|
||||
verify.append (self.ossl_path)
|
||||
verify.append ("verify")
|
||||
verify.append (filename)
|
||||
|
||||
verify = subprocess.run (verify, capture_output=True, text=True)
|
||||
print (verify.stdout)
|
||||
|
||||
else:
|
||||
print (result.stdout)
|
||||
|
||||
|
||||
class BotRelease (object):
|
||||
def __init__(self):
|
||||
print ("Initializing Packaging")
|
||||
|
||||
meson_src_root_env = "MESON_SOURCE_ROOT"
|
||||
|
||||
if meson_src_root_env in os.environ:
|
||||
os.chdir (os.environ.get (meson_src_root_env))
|
||||
|
||||
self.workDir = os.path.join (pathlib.Path ().absolute (), 'cfg')
|
||||
self.botDir = os.path.join (self.workDir, 'addons', 'yapb')
|
||||
self.distDir = os.path.join (pathlib.Path ().absolute (), 'dist');
|
||||
self.work_dir = os.path.join (pathlib.Path ().absolute (), "cfg")
|
||||
self.bot_dir = os.path.join (self.work_dir, "addons", "yapb")
|
||||
self.pkg_dir = os.path.join (pathlib.Path ().absolute (), "pkg")
|
||||
|
||||
if len (sys.argv) < 2:
|
||||
raise Exception('Missing required parameters.')
|
||||
raise Exception("Missing required parameters.")
|
||||
|
||||
self.version = sys.argv[1]
|
||||
self.artifacts = 'artifacts'
|
||||
|
||||
self.cs = CodeSign ("YaPB", "https://yapb.ru/")
|
||||
|
||||
if self.cs.has ():
|
||||
print ("Code Signing Enabled")
|
||||
else:
|
||||
print ("Code Signing Disabled")
|
||||
|
||||
os.makedirs (self.pkg_dir, exist_ok=True)
|
||||
|
||||
os.makedirs (self.distDir, exist_ok=True)
|
||||
self.pkg_win32 = os.path.join (self.pkg_dir, "yapb-{}-windows.zip".format (self.version))
|
||||
self.pkg_linux = os.path.join (self.pkg_dir, "yapb-{}-linux.tar.xz".format (self.version))
|
||||
self.pkg_macos = os.path.join (self.pkg_dir, "yapb-{}-macos.zip".format (self.version))
|
||||
|
||||
self.pkg_win32_sfx = self.pkg_win32.replace ("zip", "exe")
|
||||
self.pkg_win32_sfx_url = "https://github.com/yapb/setup/releases/latest/download/botsetup.exe"
|
||||
|
||||
self.outFileWin32 = os.path.join (self.distDir, "yapb-{}-windows.zip".format (self.version))
|
||||
self.outFileLinux = os.path.join (self.distDir, "yapb-{}-linux.tar.xz".format (self.version))
|
||||
self.outFileMacOS = os.path.join (self.distDir, "yapb-{}-macos.zip".format (self.version))
|
||||
|
||||
self.outFileWin32Setup = self.outFileWin32.replace ("zip", "exe")
|
||||
self.win32SetupUrl = "https://github.com/yapb/setup/releases/latest/download/botsetup.exe"
|
||||
|
||||
def makeDirs (self):
|
||||
def make_directories (self):
|
||||
dirs = [
|
||||
'bin',
|
||||
os.path.join ('data', 'pwf'),
|
||||
os.path.join ('data', 'train'),
|
||||
os.path.join ('data', 'graph'),
|
||||
os.path.join ('data', 'logs')
|
||||
"bin",
|
||||
os.path.join ("data", "pwf"),
|
||||
os.path.join ("data", "train"),
|
||||
os.path.join ("data", "graph"),
|
||||
os.path.join ("data", "logs")
|
||||
]
|
||||
|
||||
for dir in dirs:
|
||||
os.makedirs (os.path.join (self.botDir, dir), exist_ok=True)
|
||||
os.makedirs (os.path.join (self.bot_dir, dir), exist_ok=True)
|
||||
|
||||
def download (self, fromWhere, toFile):
|
||||
def http_pull (self, url, local_file):
|
||||
http = urllib3.PoolManager (10, headers = {"user-agent": "YaPB"})
|
||||
data = http.urlopen ('GET', fromWhere)
|
||||
data = http.urlopen ("GET", url)
|
||||
|
||||
with open (toFile, "wb") as file:
|
||||
with open (local_file, "wb") as file:
|
||||
file.write (data.data)
|
||||
|
||||
def getGraph (self, name):
|
||||
file = os.path.join (self.botDir, 'data', 'graph', "{}.graph".format (name))
|
||||
url = "http://graph.yapb.ru/graph/{}.graph".format (name);
|
||||
def get_graph_file (self, name):
|
||||
file = os.path.join (self.bot_dir, "data", "graph", "{}.graph".format (name))
|
||||
url = "http://graph.yapb.ru/graph/{}.graph".format (name)
|
||||
|
||||
if os.path.exists (file):
|
||||
return
|
||||
|
||||
self.download (url, file)
|
||||
self.http_pull (url, file)
|
||||
|
||||
def getGraphs (self):
|
||||
def get_default_graphs (self):
|
||||
print ("Downloading graphs: ")
|
||||
|
||||
files = ['as_oilrig', 'cs_747', 'cs_estate', 'cs_assault', 'cs_office',
|
||||
'cs_italy', 'cs_havana', 'cs_siege', 'cs_backalley', 'cs_militia',
|
||||
'cs_downed_cz', 'cs_havana_cz', 'cs_italy_cz', 'cs_militia_cz',
|
||||
'cs_office_cz', 'de_airstrip_cz', 'de_aztec_cz', 'de_cbble_cz',
|
||||
'de_chateau_cz', 'de_corruption_cz', 'de_dust_cz', 'de_dust2_cz',
|
||||
'de_fastline_cz', 'de_inferno_cz', 'de_piranesi_cz', 'de_prodigy_cz',
|
||||
'de_sienna_cz', 'de_stadium_cz', 'de_tides_cz', 'de_torn_cz',
|
||||
'de_truth_cz', 'de_vostok_cz', 'de_inferno', 'de_aztec', 'de_dust',
|
||||
'de_dust2', 'de_torn', 'de_storm', 'de_airstrip', 'de_piranesi',
|
||||
'de_prodigy', 'de_chateau', 'de_cbble', 'de_nuke', 'de_survivor',
|
||||
'de_vertigo', 'de_train']
|
||||
default_list = "default.graph.txt"
|
||||
self.http_pull ("http://graph.yapb.ru/DEFAULT.txt", default_list)
|
||||
|
||||
with open (default_list) as file:
|
||||
files = [line.rstrip () for line in file.readlines ()]
|
||||
|
||||
for file in files:
|
||||
print (" " + file)
|
||||
|
||||
self.getGraph (file)
|
||||
self.get_graph_file (file)
|
||||
|
||||
def unlinkBinaries (self):
|
||||
libs = ['yapb.so', 'yapb.dll', 'yapb.dylib']
|
||||
def unlink_binaries (self):
|
||||
libs = ["yapb.so", "yapb.dll", "yapb.dylib"]
|
||||
|
||||
for lib in libs:
|
||||
path = os.path.join (self.botDir, 'bin', lib)
|
||||
path = os.path.join (self.bot_dir, "bin", lib)
|
||||
|
||||
if os.path.exists (path):
|
||||
os.remove (path)
|
||||
|
||||
def copyBinary (self, path):
|
||||
shutil.copy (path, os.path.join (self.botDir, 'bin'))
|
||||
|
||||
def sign_binary (self, binary):
|
||||
if self.cs.has () and (binary.endswith ("dll") or binary.endswith ("exe")):
|
||||
print ("Signing {}".format (binary))
|
||||
self.cs.sign_file_inplace (binary)
|
||||
|
||||
def copy_binary (self, binary):
|
||||
dest_path = os.path.join (self.bot_dir, "bin", os.path.basename (binary))
|
||||
shutil.copy (binary, dest_path)
|
||||
|
||||
def zipDir (self, path, handle):
|
||||
self.sign_binary (dest_path)
|
||||
|
||||
def compress_directory (self, path, handle):
|
||||
length = len (path) + 1
|
||||
emptyDirs = []
|
||||
empty_dirs = []
|
||||
|
||||
for root, dirs, files in os.walk (path):
|
||||
emptyDirs.extend ([dir for dir in dirs if os.listdir (os.path.join (root, dir)) == []])
|
||||
empty_dirs.extend ([dir for dir in dirs if os.listdir (os.path.join (root, dir)) == []])
|
||||
|
||||
for file in files:
|
||||
filePath = os.path.join (root, file)
|
||||
handle.write (filePath, filePath[length:])
|
||||
file_path = os.path.join (root, file)
|
||||
handle.write (file_path, file_path[length:])
|
||||
|
||||
for dir in emptyDirs:
|
||||
dirPath = os.path.join (root, dir)
|
||||
for dir in empty_dirs:
|
||||
dir_path = os.path.join (root, dir)
|
||||
|
||||
zif = zipfile.ZipInfo (dirPath[length:] + "/")
|
||||
zif = zipfile.ZipInfo (dir_path[length:] + "/")
|
||||
handle.writestr (zif, "")
|
||||
|
||||
emptyDirs = []
|
||||
empty_dirs = []
|
||||
|
||||
def generateZip (self, dir):
|
||||
zipFile = zipfile.ZipFile (dir, 'w', zipfile.ZIP_DEFLATED)
|
||||
zipFile.comment = bytes (self.version, encoding = "ascii")
|
||||
def create_zip (self, dir):
|
||||
zf = zipfile.ZipFile (dir, "w", zipfile.ZIP_DEFLATED, compresslevel=9)
|
||||
zf.comment = bytes (self.version, encoding = "ascii")
|
||||
|
||||
self.zipDir (self.workDir, zipFile)
|
||||
zipFile.close ()
|
||||
self.compress_directory (self.work_dir, zf)
|
||||
zf.close ()
|
||||
|
||||
def convertZipToTXZ (self, zip, txz):
|
||||
def convert_zip_txz (self, zip, txz):
|
||||
timeshift = int ((datetime.datetime.now () - datetime.datetime.utcnow ()).total_seconds ())
|
||||
|
||||
with zipfile.ZipFile (zip) as zipf:
|
||||
with tarfile.open (txz, 'w:xz') as tarf:
|
||||
with tarfile.open (txz, "w:xz") as tarf:
|
||||
for zif in zipf.infolist ():
|
||||
tif = tarfile.TarInfo (name = zif.filename)
|
||||
tif.size = zif.file_size
|
||||
|
|
@ -135,91 +231,67 @@ class BotRelease (object):
|
|||
tarf.addfile (tarinfo = tif, fileobj = zipf.open (zif.filename))
|
||||
|
||||
os.remove (zip)
|
||||
|
||||
def generateWin32 (self):
|
||||
print ("Generating Win32 ZIP")
|
||||
|
||||
binary = os.path.join ('build_x86_win32', 'yapb.dll')
|
||||
|
||||
def install_binary (self, ext):
|
||||
lib = "yapb.{}".format (ext)
|
||||
binary = os.path.join (self.artifacts, lib)
|
||||
|
||||
if os.path.isdir (binary):
|
||||
binary = os.path.join (binary, lib)
|
||||
|
||||
if not os.path.exists (binary):
|
||||
return
|
||||
print ("Packaging failed for {}. Skipping...", lib)
|
||||
return False
|
||||
|
||||
self.unlinkBinaries ()
|
||||
self.unlink_binaries ()
|
||||
self.copy_binary (binary)
|
||||
|
||||
return True
|
||||
|
||||
self.copyBinary (binary)
|
||||
self.generateZip (self.outFileWin32)
|
||||
def create_pkg_win32 (self):
|
||||
print ("Generating Win32 ZIP")
|
||||
|
||||
self.download (self.win32SetupUrl, "botsetup.exe")
|
||||
if not self.install_binary ("dll"):
|
||||
return
|
||||
|
||||
self.create_zip (self.pkg_win32)
|
||||
self.http_pull (self.pkg_win32_sfx_url, "botsetup.exe")
|
||||
|
||||
print ("Generating Win32 EXE")
|
||||
|
||||
with open ("botsetup.exe", "rb") as sfx, open (self.outFileWin32, "rb") as zip, open (self.outFileWin32Setup, "wb") as exe:
|
||||
with open ("botsetup.exe", "rb") as sfx, open (self.pkg_win32, "rb") as zip, open (self.pkg_win32_sfx, "wb") as exe:
|
||||
exe.write (sfx.read ())
|
||||
exe.write (zip.read ())
|
||||
|
||||
self.sign_binary (self.pkg_win32_sfx)
|
||||
|
||||
def generateLinux (self):
|
||||
def create_pkg_linux (self):
|
||||
print ("Generating Linux TXZ")
|
||||
|
||||
binary = os.path.join ('build_x86_linux', 'yapb.so');
|
||||
|
||||
if not os.path.exists (binary):
|
||||
if not self.install_binary ("so"):
|
||||
return
|
||||
|
||||
self.unlinkBinaries ()
|
||||
self.copyBinary (binary)
|
||||
|
||||
tmpFile = "tmp.zip"
|
||||
tmp_file = "tmp.zip"
|
||||
|
||||
self.generateZip (tmpFile)
|
||||
self.convertZipToTXZ (tmpFile, self.outFileLinux)
|
||||
|
||||
def generateMac (self):
|
||||
self.create_zip (tmp_file)
|
||||
self.convert_zip_txz (tmp_file, self.pkg_linux)
|
||||
|
||||
def create_pkg_macos (self):
|
||||
print ("Generating macOS ZIP")
|
||||
|
||||
binary = os.path.join ('build_x86_macos', 'yapb.dylib')
|
||||
|
||||
if not os.path.exists (binary):
|
||||
if not self.install_binary ("dylib"):
|
||||
return
|
||||
|
||||
self.unlinkBinaries ()
|
||||
self.copyBinary (binary)
|
||||
|
||||
self.generateZip (self.outFileMacOS)
|
||||
self.create_zip (self.pkg_macos)
|
||||
|
||||
def generate (self):
|
||||
self.generateWin32 ()
|
||||
self.generateLinux ()
|
||||
self.generateMac ()
|
||||
|
||||
def createRelease (self, repository, version):
|
||||
print ("Creating Github Tag")
|
||||
|
||||
releases = [self.outFileLinux, self.outFileWin32, self.outFileMacOS, self.outFileWin32Setup]
|
||||
|
||||
for release in releases:
|
||||
if not os.path.exists (release):
|
||||
return
|
||||
|
||||
releaseName = "YaPB " + version
|
||||
releaseMessage = repository.get_commits()[0].commit.message
|
||||
releaseSha = repository.get_commits()[0].sha;
|
||||
def create_pkgs (self):
|
||||
self.create_pkg_linux ()
|
||||
self.create_pkg_win32 ()
|
||||
self.create_pkg_macos ()
|
||||
|
||||
ghr = repository.create_git_tag_and_release (tag = version, tag_message = version, release_name = releaseName, release_message = releaseMessage, type='commit', object = releaseSha, draft = False)
|
||||
|
||||
print ("Uploading packages to Github")
|
||||
|
||||
for release in releases:
|
||||
ghr.upload_asset( path = release, label = os.path.basename (release))
|
||||
|
||||
def uploadGithub (self):
|
||||
gh = Github (os.environ["GITHUB_TOKEN"])
|
||||
repo = gh.get_repo ("yapb/yapb")
|
||||
|
||||
self.createRelease (repo, self.version)
|
||||
|
||||
release = BotRelease ()
|
||||
|
||||
release.makeDirs ()
|
||||
release.getGraphs ()
|
||||
release.generate ()
|
||||
release.uploadGithub ()
|
||||
release.make_directories ()
|
||||
release.get_default_graphs ()
|
||||
release.create_pkgs ()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue