Python源码示例:keras.utils.Sequence()
示例1
def __init__(self, X, y, batch_size, process_fn=None):
"""A `Sequence` implementation that returns balanced `y` by undersampling majority class.
Args:
X: The numpy array of inputs.
y: The numpy array of targets.
batch_size: The generator mini-batch size.
process_fn: The preprocessing function to apply on `X`
"""
self.X = X
self.y = y
self.batch_size = batch_size
self.process_fn = process_fn or (lambda x: x)
self.pos_indices = np.where(y == 1)[0]
self.neg_indices = np.where(y == 0)[0]
self.n = min(len(self.pos_indices), len(self.neg_indices))
self._index_array = None
示例2
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例3
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例4
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例5
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例6
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例7
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例8
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例9
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例10
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例11
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例12
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例13
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例14
def test_warnings():
a = Input(shape=(3,), name='input_a')
b = Input(shape=(3,), name='input_b')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
b_2 = dp(b)
model = Model([a, b], [a_2, b_2])
optimizer = 'rmsprop'
loss = 'mse'
loss_weights = [1., 0.5]
model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
sample_weight_mode=None)
def gen_data(batch_sz):
while True:
yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
[np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
with pytest.warns(Warning) as w:
out = model.fit_generator(gen_data(4), steps_per_epoch=10, use_multiprocessing=True, workers=2)
warning_raised = any(['Sequence' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when using generator with processes.'
with pytest.warns(None) as w:
out = model.fit_generator(RandomSequence(3), steps_per_epoch=4, use_multiprocessing=True, workers=2)
assert all(['Sequence' not in str(w_.message) for w_ in w]), 'A warning was raised for Sequence.'
示例15
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例16
def __len__(self):
"""Number of batches in the Sequence."""
return int(np.ceil(len(self.samples) / self.batch_size))
示例17
def __init__(self, X, y, batch_size, process_fn=None):
"""A `Sequence` implementation that can pre-process a mini-batch via `process_fn`
Args:
X: The numpy array of inputs.
y: The numpy array of targets.
batch_size: The generator mini-batch size.
process_fn: The preprocessing function to apply on `X`
"""
self.X = X
self.y = y
self.batch_size = batch_size
self.process_fn = process_fn or (lambda x: x)