Source code

Revision control

Other Tools

1
#!/usr/bin/env python2.7
2
# This Source Code Form is subject to the terms of the Mozilla Public
3
# License, v. 2.0. If a copy of the MPL was not distributed with this
4
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6
# This script is used to create and manipulate archives containing
7
# files necessary to build Firefox on Windows (referred to as the
8
# "Windows toolchain").
9
#
10
# When updating behavior of this script, remember to update the docs
11
# in ``build/docs/toolchains.rst``.
12
13
from __future__ import absolute_import, unicode_literals
14
15
import hashlib
16
import os
17
import sys
18
19
from mozpack.files import (
20
FileFinder,
21
)
22
from mozpack.mozjar import (
23
JarWriter,
24
)
25
import mozpack.path as mozpath
26
27
SDK_RELEASE = '10.0.17134.0'
28
29
PATTERNS = [
30
{
31
'srcdir': '%(vs_path)s/DIA SDK',
32
'dstdir': 'DIA SDK',
33
'files': [
34
{
35
'pattern': 'bin/**',
36
'ignore': (
37
'bin/arm/**',
38
),
39
},
40
{
41
'pattern': 'idl/**',
42
},
43
{
44
'pattern': 'include/**',
45
},
46
{
47
'pattern': 'lib/**',
48
'ignore': (
49
'lib/arm/**',
50
),
51
},
52
],
53
},
54
{
55
'srcdir': '%(vs_path)s/VC/Tools/MSVC/14.16.27023',
56
'dstdir': 'VC',
57
'files': [
58
# ATL is needed by Breakpad.
59
{
60
'pattern': 'atlmfc/include/**',
61
},
62
{
63
'pattern': 'atlmfc/lib/arm64/atls.*',
64
},
65
{
66
'pattern': 'atlmfc/lib/x64/atls.*',
67
},
68
{
69
'pattern': 'atlmfc/lib/x86/atls.*',
70
},
71
# ARM64 PGO-instrumented builds require ARM64 pgort140.dll.
72
{
73
'pattern': 'bin/arm64/pgort140.dll',
74
},
75
{
76
'pattern': 'bin/Hostx64/**',
77
},
78
# 32-bit PGO-instrumented builds require 32-bit pgort140.dll.
79
{
80
'pattern': 'bin/Hostx86/x86/pgort140.dll',
81
},
82
{
83
'pattern': 'include/**',
84
},
85
{
86
'pattern': 'lib/**',
87
'ignore': (
88
'lib/arm64/store/**',
89
'lib/onecore/**',
90
'lib/x64/store/**',
91
'lib/x86/store/**',
92
),
93
},
94
],
95
},
96
{
97
'srcdir': '%(vs_path)s/VC/Redist/MSVC/14.16.27012',
98
'dstdir': 'VC/redist',
99
'files': [
100
{
101
'pattern': 'arm64/Microsoft.VC141.CRT/**',
102
},
103
{
104
'pattern': 'x64/Microsoft.VC141.CRT/**',
105
},
106
{
107
'pattern': 'x86/Microsoft.VC141.CRT/**',
108
},
109
],
110
},
111
{
112
'srcdir': '%(sdk_path)s',
113
'dstdir': 'SDK',
114
'files': [
115
{
116
'pattern': 'bin/%s/x64/**' % SDK_RELEASE,
117
},
118
{
119
'pattern': 'Include/%s/**' % SDK_RELEASE,
120
},
121
{
122
'pattern': 'Lib/%s/ucrt/arm64/**' % SDK_RELEASE,
123
},
124
{
125
'pattern': 'Lib/%s/ucrt/x64/**' % SDK_RELEASE,
126
},
127
{
128
'pattern': 'Lib/%s/ucrt/x86/**' % SDK_RELEASE,
129
},
130
{
131
'pattern': 'Lib/%s/um/arm64/**' % SDK_RELEASE,
132
},
133
{
134
'pattern': 'Lib/%s/um/x64/**' % SDK_RELEASE,
135
},
136
{
137
'pattern': 'Lib/%s/um/x86/**' % SDK_RELEASE,
138
},
139
{
140
'pattern': 'Redist/D3D/**',
141
},
142
{
143
'pattern': 'Redist/ucrt/DLLs/x64/**',
144
},
145
{
146
'pattern': 'Redist/ucrt/DLLs/x86/**',
147
},
148
],
149
},
150
]
151
152
153
def find_vs_paths():
154
"""Resolve source locations of files.
155
156
Returns a 2-tuple of (Visual Studio Path, SDK Path).
157
"""
158
pf = os.environ.get('ProgramFiles(x86)')
159
if not pf:
160
raise Exception('No "ProgramFiles(x86)" environment variable. '
161
'Not running on 64-bit Windows?')
162
163
vs_path = os.path.join(pf, 'Microsoft Visual Studio', '2017', 'Community')
164
if not os.path.exists(vs_path):
165
raise Exception('%s does not exist; Visual Studio 2017 not installed?' %
166
vs_path)
167
168
sdk_path = os.path.join(pf, 'Windows Kits', '10')
169
if not os.path.exists(sdk_path):
170
raise Exception('%s does not exist; Windows 10 SDK not installed?' %
171
sdk_path)
172
173
sdk_fullver_path = os.path.join(sdk_path, 'Include', SDK_RELEASE)
174
if not os.path.exists(sdk_fullver_path):
175
raise Exception('%s does not exist; Wrong SDK version installed?' %
176
sdk_fullver_path)
177
178
return vs_path, sdk_path
179
180
181
def resolve_files():
182
"""Resolve the files that constitute a standalone toolchain.
183
184
This is a generator of (dest path, file) where the destination
185
path is relative and the file instance is a BaseFile from mozpack.
186
"""
187
vs_path, sdk_path = find_vs_paths()
188
189
for entry in PATTERNS:
190
fullpath = entry['srcdir'] % {
191
'vs_path': vs_path,
192
'sdk_path': sdk_path,
193
}
194
for pattern in entry['files']:
195
finder = FileFinder(fullpath, ignore=pattern.get('ignore', []))
196
for p, f in finder.find(pattern['pattern']):
197
dstpath = '%s/%s' % (entry['dstdir'], p)
198
yield dstpath.encode('utf-8'), f
199
200
201
def resolve_files_and_hash(manifest):
202
"""Resolve files and hash their data.
203
204
This is a generator of 3-tuples of (relpath, data, mode).
205
206
As data is read, the manifest is populated with metadata.
207
Keys are set to the relative file path. Values are 2-tuples
208
of (data length, sha-256).
209
"""
210
assert manifest == {}
211
for p, f in resolve_files():
212
data = f.read()
213
214
sha256 = hashlib.sha256()
215
sha256.update(data)
216
manifest[p] = (len(data), sha256.hexdigest())
217
218
yield p, data, f.mode
219
220
221
def format_manifest(manifest):
222
"""Return formatted SHA-256 manifests as a byte strings."""
223
sha256_lines = []
224
for path, (length, sha256) in sorted(manifest.items()):
225
sha256_lines.append(b'%s\t%d\t%s' % (sha256, length, path))
226
227
# Trailing newline.
228
sha256_lines.append(b'')
229
230
return b'\n'.join(sha256_lines)
231
232
233
def write_zip(zip_path, prefix=None):
234
"""Write toolchain data to a zip file."""
235
if isinstance(prefix, unicode): # noqa Special case for Python 2
236
prefix = prefix.encode('utf-8')
237
238
with JarWriter(file=zip_path, compress_level=5) as zip:
239
manifest = {}
240
for p, data, mode in resolve_files_and_hash(manifest):
241
print(p)
242
if prefix:
243
p = mozpath.join(prefix, p)
244
245
zip.add(p, data, mode=mode)
246
247
sha256_manifest = format_manifest(manifest)
248
249
sdk_path = b'SDK_VERSION'
250
sha256_path = b'MANIFEST.SHA256'
251
if prefix:
252
sdk_path = mozpath.join(prefix, sdk_path)
253
sha256_path = mozpath.join(prefix, sha256_path)
254
255
zip.add(sdk_path, SDK_RELEASE.encode('utf-8'))
256
zip.add(sha256_path, sha256_manifest)
257
258
259
if __name__ == '__main__':
260
if len(sys.argv) != 3:
261
print('usage: %s create-zip <path-prefix>' % sys.argv[0])
262
sys.exit(1)
263
264
assert sys.argv[1] == 'create-zip'
265
prefix = os.path.basename(sys.argv[2])
266
destzip = '%s.zip' % sys.argv[2]
267
write_zip(destzip, prefix=prefix)
268
269
sha1 = hashlib.sha1()
270
sha256 = hashlib.sha256()
271
sha512 = hashlib.sha512()
272
273
with open(destzip, 'rb') as fh:
274
data = fh.read()
275
sha1.update(data)
276
sha256.update(data)
277
sha512.update(data)
278
279
print('Hashes of %s (size=%d)' % (destzip, len(data)))
280
print('SHA-1: %s' % sha1.hexdigest())
281
print('SHA-256: %s' % sha256.hexdigest())
282
print('SHA-512: %s' % sha512.hexdigest())